oh-my-knowledge 0.20.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -4
- package/README.zh.md +8 -4
- package/dist/src/analysis/coverage-analyzer.d.ts +1 -1
- package/dist/src/analysis/coverage-analyzer.d.ts.map +1 -1
- package/dist/src/analysis/failure-clusterer.d.ts +1 -1
- package/dist/src/analysis/failure-clusterer.d.ts.map +1 -1
- package/dist/src/analysis/gap-analyzer.d.ts +2 -2
- package/dist/src/analysis/gap-analyzer.d.ts.map +1 -1
- package/dist/src/analysis/gap-analyzer.js +4 -4
- package/dist/src/analysis/hedging-classifier.d.ts +1 -1
- package/dist/src/analysis/hedging-classifier.d.ts.map +1 -1
- package/dist/src/analysis/report-diagnostics.d.ts +1 -1
- package/dist/src/analysis/report-diagnostics.d.ts.map +1 -1
- package/dist/src/analysis/report-diagnostics.js +7 -7
- package/dist/src/analysis/sample-diagnostics.d.ts +1 -1
- package/dist/src/analysis/sample-diagnostics.d.ts.map +1 -1
- package/dist/src/analysis/sample-diagnostics.js +15 -15
- package/dist/src/authoring/evolver.d.ts +1 -1
- package/dist/src/authoring/evolver.d.ts.map +1 -1
- package/dist/src/authoring/evolver.js +5 -5
- package/dist/src/authoring/evolver.js.map +1 -1
- package/dist/src/authoring/generator.d.ts +1 -1
- package/dist/src/authoring/generator.d.ts.map +1 -1
- package/dist/src/authoring/generator.js +8 -8
- package/dist/src/authoring/generator.js.map +1 -1
- package/dist/src/cli/i18n-dict.d.ts +56 -0
- package/dist/src/cli/i18n-dict.d.ts.map +1 -0
- package/dist/src/cli/i18n-dict.js +912 -0
- package/dist/src/cli/i18n-dict.js.map +1 -0
- package/dist/src/cli/i18n.d.ts +24 -0
- package/dist/src/cli/i18n.d.ts.map +1 -0
- package/dist/src/cli/i18n.js +53 -0
- package/dist/src/cli/i18n.js.map +1 -0
- package/dist/src/cli.js +279 -408
- package/dist/src/cli.js.map +1 -1
- package/dist/src/eval-core/cache.d.ts +1 -1
- package/dist/src/eval-core/cache.d.ts.map +1 -1
- package/dist/src/eval-core/dependency-checker.d.ts +16 -1
- package/dist/src/eval-core/dependency-checker.d.ts.map +1 -1
- package/dist/src/eval-core/dependency-checker.js +79 -4
- package/dist/src/eval-core/dependency-checker.js.map +1 -1
- package/dist/src/eval-core/evaluation-execution.d.ts +3 -3
- package/dist/src/eval-core/evaluation-execution.d.ts.map +1 -1
- package/dist/src/eval-core/evaluation-execution.js +1 -1
- package/dist/src/eval-core/evaluation-execution.js.map +1 -1
- package/dist/src/eval-core/evaluation-job.d.ts +2 -2
- package/dist/src/eval-core/evaluation-job.d.ts.map +1 -1
- package/dist/src/eval-core/evaluation-reporting.d.ts +1 -1
- package/dist/src/eval-core/evaluation-reporting.d.ts.map +1 -1
- package/dist/src/eval-core/execution-strategy.d.ts +1 -1
- package/dist/src/eval-core/execution-strategy.d.ts.map +1 -1
- package/dist/src/eval-core/layer-gates.d.ts +17 -0
- package/dist/src/eval-core/layer-gates.d.ts.map +1 -0
- package/dist/src/eval-core/{ci-gates.js → layer-gates.js} +5 -5
- package/dist/src/eval-core/layer-gates.js.map +1 -0
- package/dist/src/eval-core/schema.d.ts +1 -1
- package/dist/src/eval-core/schema.d.ts.map +1 -1
- package/dist/src/eval-core/schema.js +1 -1
- package/dist/src/eval-core/task-planner.d.ts +1 -1
- package/dist/src/eval-core/task-planner.d.ts.map +1 -1
- package/dist/src/eval-core/task-planner.js +1 -1
- package/dist/src/eval-core/task-planner.js.map +1 -1
- package/dist/src/eval-core/verdict.d.ts +5 -2
- package/dist/src/eval-core/verdict.d.ts.map +1 -1
- package/dist/src/eval-core/verdict.js +47 -8
- package/dist/src/eval-core/verdict.js.map +1 -1
- package/dist/src/eval-workflows/each-evaluation-workflow.d.ts +7 -7
- package/dist/src/eval-workflows/each-evaluation-workflow.d.ts.map +1 -1
- package/dist/src/eval-workflows/evaluation-pipeline.d.ts +18 -3
- package/dist/src/eval-workflows/evaluation-pipeline.d.ts.map +1 -1
- package/dist/src/eval-workflows/evaluation-pipeline.js +37 -0
- package/dist/src/eval-workflows/evaluation-pipeline.js.map +1 -1
- package/dist/src/eval-workflows/evaluation-preparation.d.ts +5 -5
- package/dist/src/eval-workflows/evaluation-preparation.d.ts.map +1 -1
- package/dist/src/eval-workflows/run-evaluation.d.ts +3 -3
- package/dist/src/eval-workflows/run-evaluation.d.ts.map +1 -1
- package/dist/src/eval-workflows/run-evaluation.js +9 -3
- package/dist/src/eval-workflows/run-evaluation.js.map +1 -1
- package/dist/src/executors/anthropic-api.d.ts +1 -1
- package/dist/src/executors/anthropic-api.d.ts.map +1 -1
- package/dist/src/executors/anthropic-api.js +1 -1
- package/dist/src/executors/anthropic-api.js.map +1 -1
- package/dist/src/executors/claude-cli.d.ts +1 -1
- package/dist/src/executors/claude-cli.d.ts.map +1 -1
- package/dist/src/executors/claude-sdk-trace.d.ts +1 -1
- package/dist/src/executors/claude-sdk-trace.d.ts.map +1 -1
- package/dist/src/executors/claude-sdk.d.ts +1 -1
- package/dist/src/executors/claude-sdk.d.ts.map +1 -1
- package/dist/src/executors/gemini.d.ts +1 -1
- package/dist/src/executors/gemini.d.ts.map +1 -1
- package/dist/src/executors/index.d.ts +1 -1
- package/dist/src/executors/index.d.ts.map +1 -1
- package/dist/src/executors/openai-api.d.ts +1 -1
- package/dist/src/executors/openai-api.d.ts.map +1 -1
- package/dist/src/executors/openai-api.js +1 -1
- package/dist/src/executors/openai-api.js.map +1 -1
- package/dist/src/executors/openai-cli.d.ts +1 -1
- package/dist/src/executors/openai-cli.d.ts.map +1 -1
- package/dist/src/executors/script.d.ts +1 -1
- package/dist/src/executors/script.d.ts.map +1 -1
- package/dist/src/executors/shared.d.ts +1 -1
- package/dist/src/executors/shared.d.ts.map +1 -1
- package/dist/src/grading/assertions.d.ts +1 -1
- package/dist/src/grading/assertions.d.ts.map +1 -1
- package/dist/src/grading/debias-validate.d.ts +1 -1
- package/dist/src/grading/debias-validate.d.ts.map +1 -1
- package/dist/src/grading/debias-validate.js +3 -3
- package/dist/src/grading/gold-cli.d.ts +1 -1
- package/dist/src/grading/gold-cli.d.ts.map +1 -1
- package/dist/src/grading/gold-cli.js +3 -3
- package/dist/src/grading/gold-cli.js.map +1 -1
- package/dist/src/grading/index.d.ts +1 -1
- package/dist/src/grading/index.d.ts.map +1 -1
- package/dist/src/grading/judge.d.ts +1 -1
- package/dist/src/grading/judge.d.ts.map +1 -1
- package/dist/src/grading/layered-scores.d.ts +1 -1
- package/dist/src/grading/layered-scores.d.ts.map +1 -1
- package/dist/src/inputs/eval-config.d.ts +1 -1
- package/dist/src/inputs/eval-config.d.ts.map +1 -1
- package/dist/src/inputs/eval-config.js +18 -18
- package/dist/src/inputs/eval-config.js.map +1 -1
- package/dist/src/inputs/load-samples.d.ts +1 -1
- package/dist/src/inputs/load-samples.d.ts.map +1 -1
- package/dist/src/inputs/load-samples.js +4 -4
- package/dist/src/inputs/load-samples.js.map +1 -1
- package/dist/src/inputs/mcp-resolver.d.ts +1 -1
- package/dist/src/inputs/mcp-resolver.d.ts.map +1 -1
- package/dist/src/inputs/mcp-resolver.js +4 -4
- package/dist/src/inputs/mcp-resolver.js.map +1 -1
- package/dist/src/inputs/skill-loader.d.ts +1 -1
- package/dist/src/inputs/skill-loader.d.ts.map +1 -1
- package/dist/src/inputs/skill-loader.js +12 -6
- package/dist/src/inputs/skill-loader.js.map +1 -1
- package/dist/src/inputs/url-fetcher.d.ts +1 -1
- package/dist/src/inputs/url-fetcher.d.ts.map +1 -1
- package/dist/src/inputs/url-fetcher.js +2 -2
- package/dist/src/inputs/url-fetcher.js.map +1 -1
- package/dist/src/observability/skill-health-analyzer.d.ts +1 -1
- package/dist/src/observability/skill-health-analyzer.d.ts.map +1 -1
- package/dist/src/observability/trace-adapter.d.ts +1 -1
- package/dist/src/observability/trace-adapter.d.ts.map +1 -1
- package/dist/src/renderer/html-renderer.d.ts +1 -1
- package/dist/src/renderer/html-renderer.d.ts.map +1 -1
- package/dist/src/renderer/html-renderer.js +43 -2
- package/dist/src/renderer/html-renderer.js.map +1 -1
- package/dist/src/renderer/layout.d.ts +1 -1
- package/dist/src/renderer/layout.d.ts.map +1 -1
- package/dist/src/renderer/layout.js +93 -1
- package/dist/src/renderer/layout.js.map +1 -1
- package/dist/src/renderer/skill-health-renderer.d.ts +1 -1
- package/dist/src/renderer/skill-health-renderer.d.ts.map +1 -1
- package/dist/src/renderer/summary.d.ts +4 -1
- package/dist/src/renderer/summary.d.ts.map +1 -1
- package/dist/src/renderer/summary.js +115 -54
- package/dist/src/renderer/summary.js.map +1 -1
- package/dist/src/renderer/table.d.ts +1 -1
- package/dist/src/renderer/table.d.ts.map +1 -1
- package/dist/src/renderer/trends.d.ts +1 -1
- package/dist/src/renderer/trends.d.ts.map +1 -1
- package/dist/src/server/job-store.d.ts +1 -1
- package/dist/src/server/job-store.d.ts.map +1 -1
- package/dist/src/server/report-server.d.ts +1 -1
- package/dist/src/server/report-server.d.ts.map +1 -1
- package/dist/src/server/report-server.js +5 -5
- package/dist/src/server/report-server.js.map +1 -1
- package/dist/src/server/report-store.d.ts +1 -1
- package/dist/src/server/report-store.d.ts.map +1 -1
- package/dist/src/types/eval.d.ts +240 -0
- package/dist/src/types/eval.d.ts.map +1 -0
- package/dist/src/types/eval.js +2 -0
- package/dist/src/types/eval.js.map +1 -0
- package/dist/src/types/executor.d.ts +65 -0
- package/dist/src/types/executor.d.ts.map +1 -0
- package/dist/src/types/executor.js +2 -0
- package/dist/src/types/executor.js.map +1 -0
- package/dist/src/types/index.d.ts +7 -0
- package/dist/src/types/index.d.ts.map +1 -0
- package/dist/src/types/index.js +7 -0
- package/dist/src/types/index.js.map +1 -0
- package/dist/src/types/judge.d.ts +98 -0
- package/dist/src/types/judge.d.ts.map +1 -0
- package/dist/src/types/judge.js +2 -0
- package/dist/src/types/judge.js.map +1 -0
- package/dist/src/types/report.d.ts +390 -0
- package/dist/src/types/report.d.ts.map +1 -0
- package/dist/src/types/report.js +2 -0
- package/dist/src/types/report.js.map +1 -0
- package/dist/src/types/shared.d.ts +2 -0
- package/dist/src/types/shared.d.ts.map +1 -0
- package/dist/src/types/shared.js +2 -0
- package/dist/src/types/shared.js.map +1 -0
- package/dist/src/types/storage.d.ts +21 -0
- package/dist/src/types/storage.d.ts.map +1 -0
- package/dist/src/types/storage.js +2 -0
- package/dist/src/types/storage.js.map +1 -0
- package/package.json +1 -1
- package/dist/src/eval-core/ci-gates.d.ts +0 -17
- package/dist/src/eval-core/ci-gates.d.ts.map +0 -1
- package/dist/src/eval-core/ci-gates.js.map +0 -1
- package/dist/src/types.d.ts +0 -800
- package/dist/src/types.d.ts.map +0 -1
- package/dist/src/types.js +0 -2
- package/dist/src/types.js.map +0 -1
package/dist/src/cli.js
CHANGED
|
@@ -4,6 +4,7 @@ import { resolve } from 'node:path';
|
|
|
4
4
|
import { homedir } from 'node:os';
|
|
5
5
|
import { join } from 'node:path';
|
|
6
6
|
import { existsSync } from 'node:fs';
|
|
7
|
+
import { tCli, getCliLang, parseLangFromArgv, langFromArgv } from './cli/i18n.js';
|
|
7
8
|
import { discoverVariants, parseVariantCwd } from './inputs/skill-loader.js';
|
|
8
9
|
import { loadEvalConfig, configVariantsToSpecs } from './inputs/eval-config.js';
|
|
9
10
|
// ---------------------------------------------------------------------------
|
|
@@ -14,7 +15,15 @@ const DEFAULT_REPORTS_DIR = join(homedir(), '.oh-my-knowledge', 'reports');
|
|
|
14
15
|
// Defaults are applied inside parseRunConfig (after config-file merge) so that
|
|
15
16
|
// CLI `undefined` can be reliably distinguished from "user passed the default value".
|
|
16
17
|
// Priority order resolved in parseRunConfig: CLI arg > --config file > hard-coded default.
|
|
18
|
+
/**
|
|
19
|
+
* 所有子命令都接受的通用 flag。新增 --lang 让 parseArgs strict:false 模式下
|
|
20
|
+
* 仍能把值类型化到 values.lang 上(否则未声明的 flag 会被丢弃)。
|
|
21
|
+
*/
|
|
22
|
+
const COMMON_OPTIONS = {
|
|
23
|
+
lang: { type: 'string' },
|
|
24
|
+
};
|
|
17
25
|
const RUN_OPTIONS = {
|
|
26
|
+
...COMMON_OPTIONS,
|
|
18
27
|
samples: { type: 'string' },
|
|
19
28
|
'skill-dir': { type: 'string' },
|
|
20
29
|
control: { type: 'string' },
|
|
@@ -172,149 +181,9 @@ function parseRunConfig(argv, extraOptions = {}) {
|
|
|
172
181
|
};
|
|
173
182
|
}
|
|
174
183
|
// ---------------------------------------------------------------------------
|
|
175
|
-
// Help text
|
|
176
|
-
// ---------------------------------------------------------------------------
|
|
177
|
-
const HELP = `
|
|
178
|
-
oh-my-knowledge — Knowledge artifact evaluation toolkit
|
|
179
|
-
|
|
180
|
-
Usage:
|
|
181
|
-
omk bench run [options] Run an evaluation
|
|
182
|
-
omk bench report [options] Start the report server
|
|
183
|
-
omk bench ci [options] Run evaluation and exit with pass/fail code
|
|
184
|
-
omk bench init [dir] Scaffold a new eval project
|
|
185
|
-
omk bench gen-samples [skill] Generate eval-samples from skill content
|
|
186
|
-
omk bench diff <id1> <id2> Compare two evaluation reports
|
|
187
|
-
omk bench evolve <skill> Self-improve a skill through iterative evaluation
|
|
188
|
-
|
|
189
|
-
omk analyze <dir> Analyze cc session trace(s), produce skill 健康度日报 (v0.18)
|
|
190
|
-
|
|
191
|
-
Options for "bench run":
|
|
192
|
-
|
|
193
|
-
--samples <path> Sample file (default: eval-samples.json)
|
|
194
|
-
--skill-dir <path> Skill definitions directory (default: skills)
|
|
195
|
-
--control <expr> Control-group variant expression (experiment role = control)
|
|
196
|
-
--treatment <v1,v2> Treatment-group variant expressions (comma-separated; role = treatment)
|
|
197
|
-
Each variant expression resolves to an artifact and optional runtime context:
|
|
198
|
-
"baseline" — bare model, no artifact injected
|
|
199
|
-
"git:name" — artifact from last commit
|
|
200
|
-
"git:ref:name" — artifact from specific commit
|
|
201
|
-
path with "/" — artifact from file directly (e.g. ./v1.md)
|
|
202
|
-
"name@/cwd" — attach runtime context / cwd
|
|
203
|
-
At least one of --control / --treatment must be provided.
|
|
204
|
-
--config <path> YAML/JSON config file (evaluation-as-code).
|
|
205
|
-
Declares samples + variants + model + executor in one file.
|
|
206
|
-
CLI flags override config fields when both are provided.
|
|
207
|
-
Relative paths inside the config are resolved against its directory.
|
|
208
|
-
--model <name> Model under test (default: sonnet)
|
|
209
|
-
--judge-model <name> Judge model (default: haiku)
|
|
210
|
-
--output-dir <path> Report output directory (default: ~/.oh-my-knowledge/reports/)
|
|
211
|
-
--no-judge Skip LLM judging
|
|
212
|
-
--no-cache Disable result caching
|
|
213
|
-
--dry-run Preview tasks without executing
|
|
214
|
-
--blind Blind A/B mode: hide variant names in report
|
|
215
|
-
--concurrency <n> Number of parallel tasks (default: 1)
|
|
216
|
-
--timeout <seconds> Executor timeout per task in seconds (default: 120)
|
|
217
|
-
--repeat <n> Run evaluation N times for variance analysis (default: 1)
|
|
218
|
-
--judge-repeat <n> Call LLM judge N times per (sample × dimension) for self-
|
|
219
|
-
consistency (default: 1). High stddev across runs = the
|
|
220
|
-
judge is unstable on this rubric and the score is noisy.
|
|
221
|
-
--judge-models <list> Multi-judge ensemble. Comma-separated executor:model pairs,
|
|
222
|
-
e.g. claude:opus,openai:gpt-4o,gemini:pro. Each judge scores
|
|
223
|
-
every (sample × dimension); report includes per-judge break-
|
|
224
|
-
down + Pearson/MAD inter-judge agreement. Refutes "Claude
|
|
225
|
-
judge Claude same-modality bias" critique. Combines with
|
|
226
|
-
--judge-repeat. Cost ~ N_judges × N_repeat × N_samples.
|
|
227
|
-
--bootstrap Compute bootstrap confidence intervals (distribution-free,
|
|
228
|
-
preferred over t-interval for ordinal LLM scores). Adds
|
|
229
|
-
per-variant CI on the mean + pairwise CI on treatment-vs-
|
|
230
|
-
control difference (significant=0 outside CI). Reports both
|
|
231
|
-
t-interval and bootstrap so old tooling still works.
|
|
232
|
-
--bootstrap-samples <n> Number of bootstrap resamples (default 1000). N>10000
|
|
233
|
-
triggers a stderr warning about runtime cost.
|
|
234
|
-
--retry <n> Retry failed tasks up to N times with exponential backoff (default: 0)
|
|
235
|
-
--resume <report-id> Resume from a previous report, skipping completed tasks
|
|
236
|
-
--executor <name> Executor: claude, openai, gemini, anthropic-api, openai-api,
|
|
237
|
-
or any shell command (e.g. "python my_provider.py")
|
|
238
|
-
--judge-executor <name> Executor for LLM judge (default: same as --executor)
|
|
239
|
-
--each Evaluate each skill independently against baseline
|
|
240
|
-
Requires {name}.eval-samples.json paired with each skill
|
|
241
|
-
--skip-preflight Skip model connectivity check before evaluation
|
|
242
|
-
--mcp-config <path> MCP config file for URL fetching via MCP servers
|
|
243
|
-
(default: .mcp.json in current directory)
|
|
244
|
-
--no-serve Skip auto-starting report server after evaluation
|
|
245
|
-
--verbose Print detailed progress for each sample (exec result, grading phases)
|
|
246
|
-
--layered-stats Expand the three-layer (fact/behavior/judge) independent
|
|
247
|
-
significance breakdown in the HTML report by default.
|
|
248
|
-
Without this flag, the breakdown is collapsed behind a
|
|
249
|
-
click-to-expand summary under each comparison.
|
|
250
|
-
|
|
251
|
-
Options for "bench ci":
|
|
252
|
-
(same as "bench run", plus:)
|
|
253
|
-
--threshold <number> Minimum score to pass, applied INDEPENDENTLY to each of
|
|
254
|
-
the three layers (fact / behavior / LLM judge). ANY
|
|
255
|
-
layer below threshold fails the gate — this prevents
|
|
256
|
-
composite averaging from masking a single-layer collapse.
|
|
257
|
-
Default: 3.5. If all three layers are absent (no
|
|
258
|
-
assertions and no rubric defined in eval-samples), the
|
|
259
|
-
gate FAILS with a configuration hint — no composite fallback.
|
|
260
|
-
|
|
261
|
-
Options for "bench report":
|
|
262
|
-
--port <number> Server port (default: 7799)
|
|
263
|
-
--reports-dir <path> Reports directory (default: ~/.oh-my-knowledge/reports/)
|
|
264
|
-
--export <id> Export report as standalone HTML file
|
|
265
|
-
--dev Dev mode: auto-restart on lib/ file changes
|
|
266
|
-
|
|
267
|
-
Options for "bench gen-samples":
|
|
268
|
-
--each Generate for all skills missing eval-samples
|
|
269
|
-
--count <n> Number of samples to generate per skill (default: 5)
|
|
270
|
-
--model <name> Model for generation (default: sonnet)
|
|
271
|
-
--skill-dir <path> Skill directory (default: skills), used with --each
|
|
272
|
-
|
|
273
|
-
Options for "analyze":
|
|
274
|
-
<dir> Input: cc session JSONL file / dir (e.g. ~/.claude/projects/<slug>)
|
|
275
|
-
--kb <path> Knowledge base root (default: auto-infer from trace cwd)
|
|
276
|
-
--last <duration> Time window like "7d" / "30d" (default: all)
|
|
277
|
-
--from <iso> Window start (ISO8601), takes precedence over --last
|
|
278
|
-
--to <iso> Window end (ISO8601), takes precedence over --last
|
|
279
|
-
--skills <n1,n2,...> Whitelist skills to analyze (default: all)
|
|
280
|
-
--output-dir <path> Output dir (default: ~/.oh-my-knowledge/analyses/)
|
|
281
|
-
|
|
282
|
-
Options for "bench evolve":
|
|
283
|
-
--rounds <n> Maximum evolution rounds (default: 5)
|
|
284
|
-
--target <score> Stop early when score reaches this threshold
|
|
285
|
-
--samples <path> Sample file (default: eval-samples.json)
|
|
286
|
-
--model <name> Model under test (default: sonnet)
|
|
287
|
-
--judge-model <name> Judge model (default: haiku)
|
|
288
|
-
--improve-model <name> Model for generating improvements (default: sonnet)
|
|
289
|
-
--concurrency <n> Parallel eval tasks (default: 1)
|
|
290
|
-
--timeout <seconds> Executor timeout per task in seconds (default: 120)
|
|
291
|
-
--executor <name> Executor to use (default: claude)
|
|
292
|
-
|
|
293
|
-
Examples:
|
|
294
|
-
omk bench run --control v1 --treatment v2
|
|
295
|
-
omk bench run --control baseline --treatment my-skill
|
|
296
|
-
omk bench run --control git:my-skill --treatment my-skill
|
|
297
|
-
omk bench run --control ./old-skill.md --treatment ./new-skill.md
|
|
298
|
-
omk bench run --control baseline --treatment v1,v2,v3
|
|
299
|
-
omk bench run --config eval.yaml
|
|
300
|
-
omk bench run --config eval.yaml --model sonnet-4.6 # CLI overrides config
|
|
301
|
-
omk bench run --each
|
|
302
|
-
omk bench run --dry-run
|
|
303
|
-
omk bench report --port 8080
|
|
304
|
-
omk bench report --export v1-vs-v2-20260326-1832
|
|
305
|
-
omk bench init my-eval
|
|
306
|
-
omk bench gen-samples skills/my-skill.md
|
|
307
|
-
omk bench gen-samples --each
|
|
308
|
-
omk bench diff <report-id-1> <report-id-2>
|
|
309
|
-
omk bench evolve skills/my-skill.md --rounds 5
|
|
310
|
-
omk analyze ~/.claude/projects/-Users-lizhiyao-Documents-oh-my-knowledge
|
|
311
|
-
omk analyze ~/.claude/projects/my-project --last 7d --kb /path/to/project
|
|
312
|
-
omk analyze ~/.claude/projects/my-project --skills audit,polish
|
|
313
|
-
`.trim();
|
|
314
|
-
// ---------------------------------------------------------------------------
|
|
315
184
|
// Update check
|
|
316
185
|
// ---------------------------------------------------------------------------
|
|
317
|
-
async function checkUpdate() {
|
|
186
|
+
async function checkUpdate(lang) {
|
|
318
187
|
try {
|
|
319
188
|
const { readFileSync } = await import('node:fs');
|
|
320
189
|
const { fileURLToPath } = await import('node:url');
|
|
@@ -327,19 +196,22 @@ async function checkUpdate() {
|
|
|
327
196
|
return;
|
|
328
197
|
const data = await res.json();
|
|
329
198
|
if (data.version && data.version !== pkg.version) {
|
|
330
|
-
process.stderr.write(
|
|
199
|
+
process.stderr.write(tCli('cli.update.new_version_available', lang, {
|
|
200
|
+
old: pkg.version, new: data.version, pkg: pkg.name,
|
|
201
|
+
}));
|
|
331
202
|
}
|
|
332
203
|
}
|
|
333
|
-
catch { /*
|
|
204
|
+
catch { /* 静默失败,不影响正常使用 */ }
|
|
334
205
|
}
|
|
335
206
|
// ---------------------------------------------------------------------------
|
|
336
207
|
// Main
|
|
337
208
|
// ---------------------------------------------------------------------------
|
|
338
209
|
async function main() {
|
|
339
|
-
|
|
210
|
+
const lang = getCliLang(parseLangFromArgv(process.argv));
|
|
211
|
+
checkUpdate(lang);
|
|
340
212
|
const [domain, command, ...rest] = process.argv.slice(2);
|
|
341
213
|
if (!domain || domain === '--help' || domain === '-h') {
|
|
342
|
-
console.log(
|
|
214
|
+
console.log(tCli('cli.help.main', lang).trim());
|
|
343
215
|
process.exit(0);
|
|
344
216
|
}
|
|
345
217
|
if (domain === 'analyze') {
|
|
@@ -348,11 +220,11 @@ async function main() {
|
|
|
348
220
|
return;
|
|
349
221
|
}
|
|
350
222
|
if (domain !== 'bench') {
|
|
351
|
-
console.error(
|
|
223
|
+
console.error(tCli('cli.common.unknown_domain', lang, { domain }));
|
|
352
224
|
process.exit(1);
|
|
353
225
|
}
|
|
354
226
|
if (!command || command === '--help' || command === '-h') {
|
|
355
|
-
console.log(
|
|
227
|
+
console.log(tCli('cli.help.main', lang).trim());
|
|
356
228
|
process.exit(0);
|
|
357
229
|
}
|
|
358
230
|
switch (command) {
|
|
@@ -365,8 +237,8 @@ async function main() {
|
|
|
365
237
|
case 'init':
|
|
366
238
|
await handleInit(rest);
|
|
367
239
|
break;
|
|
368
|
-
case '
|
|
369
|
-
await
|
|
240
|
+
case 'gate':
|
|
241
|
+
await handleGate(rest);
|
|
370
242
|
break;
|
|
371
243
|
case 'gen-samples':
|
|
372
244
|
await handleGenSamples(rest);
|
|
@@ -396,58 +268,75 @@ async function main() {
|
|
|
396
268
|
await handleFailures(rest);
|
|
397
269
|
break;
|
|
398
270
|
default:
|
|
399
|
-
console.error(
|
|
271
|
+
console.error(tCli('cli.common.unknown_bench_command', lang, { command }));
|
|
400
272
|
process.exit(1);
|
|
401
273
|
}
|
|
402
274
|
}
|
|
403
275
|
// ---------------------------------------------------------------------------
|
|
404
276
|
// Progress callback
|
|
405
277
|
// ---------------------------------------------------------------------------
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
process.stderr.write(`[${completed}/${total}] ${sample_id}/${variant} ❌ ${error}\n`);
|
|
417
|
-
return;
|
|
418
|
-
}
|
|
419
|
-
if (phase === 'start') {
|
|
420
|
-
process.stderr.write(`[${completed}/${total}] ${sample_id}/${variant} ⏳ 执行中...\n`);
|
|
421
|
-
}
|
|
422
|
-
else if (phase === 'exec_done') {
|
|
423
|
-
const costInfo = costUSD != null && costUSD > 0 ? ` $${costUSD.toFixed(4)}` : '';
|
|
424
|
-
process.stderr.write(`[${completed}/${total}] ${sample_id}/${variant} 执行完成 ${durationMs}ms ${inputTokens}+${outputTokens} tokens${costInfo}\n`);
|
|
425
|
-
if (outputPreview) {
|
|
426
|
-
process.stderr.write(` 输出预览: ${outputPreview.slice(0, 150).replace(/\n/g, ' ')}\n`);
|
|
278
|
+
/**
|
|
279
|
+
* Factory: 闭住 lang, 返回 onProgress callback。evaluation engine 回调时不传
|
|
280
|
+
* 上下文, 所以 lang 必须在 handler 入口处通过 closure 传进来。
|
|
281
|
+
*/
|
|
282
|
+
function makeOnProgress(lang) {
|
|
283
|
+
return ({ phase, completed, total, sample_id, variant, durationMs, inputTokens, outputTokens, costUSD, score, outputPreview, judgePhase: _judgePhase, judgeDim, skipped, attempt, maxAttempts, error, }) => {
|
|
284
|
+
const ctx = { i: completed ?? '', n: total ?? '', sample: sample_id ?? '', variant: variant ?? '' };
|
|
285
|
+
if (phase === 'preflight') {
|
|
286
|
+
process.stderr.write(tCli('cli.progress.preflight_starting', lang));
|
|
287
|
+
return;
|
|
427
288
|
}
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
if (
|
|
439
|
-
process.stderr.write(
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
289
|
+
if (phase === 'retry') {
|
|
290
|
+
process.stderr.write(tCli('cli.progress.sample_retry', lang, {
|
|
291
|
+
...ctx, attempt: attempt ?? '', max: maxAttempts ?? '',
|
|
292
|
+
}));
|
|
293
|
+
return;
|
|
294
|
+
}
|
|
295
|
+
if (phase === 'error') {
|
|
296
|
+
process.stderr.write(tCli('cli.progress.sample_error', lang, { ...ctx, error: error ?? '' }));
|
|
297
|
+
return;
|
|
298
|
+
}
|
|
299
|
+
if (phase === 'start') {
|
|
300
|
+
process.stderr.write(tCli('cli.progress.sample_executing', lang, ctx));
|
|
301
|
+
}
|
|
302
|
+
else if (phase === 'exec_done') {
|
|
303
|
+
const cost = costUSD != null && costUSD > 0 ? ` $${costUSD.toFixed(4)}` : '';
|
|
304
|
+
process.stderr.write(tCli('cli.progress.sample_exec_done', lang, {
|
|
305
|
+
...ctx, ms: durationMs ?? '', input: inputTokens ?? '', output: outputTokens ?? '', cost,
|
|
306
|
+
}));
|
|
307
|
+
if (outputPreview) {
|
|
308
|
+
process.stderr.write(tCli('cli.progress.output_preview', lang, {
|
|
309
|
+
preview: outputPreview.slice(0, 150).replace(/\n/g, ' '),
|
|
310
|
+
}));
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
else if (phase === 'grading') {
|
|
314
|
+
const dim = judgeDim ? ` [${judgeDim}]` : '';
|
|
315
|
+
process.stderr.write(tCli('cli.progress.judging', lang, { ...ctx, dim }));
|
|
316
|
+
}
|
|
317
|
+
else if (phase === 'judge_done') {
|
|
318
|
+
const dim = judgeDim ? ` [${judgeDim}]` : '';
|
|
319
|
+
process.stderr.write(tCli('cli.progress.judged', lang, { ...ctx, dim, score: score ?? '' }));
|
|
320
|
+
}
|
|
321
|
+
else if (phase === 'done' && skipped) {
|
|
322
|
+
if (sample_id)
|
|
323
|
+
process.stderr.write(tCli('cli.progress.skipped', lang, ctx));
|
|
324
|
+
}
|
|
325
|
+
else {
|
|
326
|
+
const cost = costUSD != null && costUSD > 0 ? ` $${costUSD.toFixed(4)}` : '';
|
|
327
|
+
const scoreInfo = typeof score === 'number' ? ` score=${score}` : '';
|
|
328
|
+
process.stderr.write(tCli('cli.progress.sample_done', lang, {
|
|
329
|
+
...ctx, ms: durationMs ?? '', input: inputTokens ?? '', output: outputTokens ?? '',
|
|
330
|
+
cost, score: scoreInfo,
|
|
331
|
+
}));
|
|
332
|
+
}
|
|
333
|
+
};
|
|
446
334
|
}
|
|
447
335
|
// ---------------------------------------------------------------------------
|
|
448
336
|
// handleRun
|
|
449
337
|
// ---------------------------------------------------------------------------
|
|
450
338
|
async function handleRun(argv) {
|
|
339
|
+
const lang = langFromArgv(argv);
|
|
451
340
|
const { values, config } = parseRunConfig(argv, {
|
|
452
341
|
blind: { type: 'boolean', default: false },
|
|
453
342
|
repeat: { type: 'string', default: '1' },
|
|
@@ -463,26 +352,26 @@ async function handleRun(argv) {
|
|
|
463
352
|
});
|
|
464
353
|
const { runEvaluation, runMultiple, runEachEvaluation } = await import('./eval-workflows/run-evaluation.js');
|
|
465
354
|
config.blind = values.blind;
|
|
466
|
-
config.onProgress =
|
|
467
|
-
// --repeat
|
|
468
|
-
// 提前到 --each
|
|
355
|
+
config.onProgress = makeOnProgress(lang);
|
|
356
|
+
// --repeat 输入校验: 非 ≥1 整数时提示并钳到 1, 不静默掩盖用户错字 / 极端输入。
|
|
357
|
+
// 提前到 --each 分支之前, 保证 each 模式也能读到 repeat (曾经 bug: --each 吞 --repeat)。
|
|
469
358
|
const repeatRaw = values.repeat;
|
|
470
359
|
const parsedRepeat = repeatRaw !== undefined ? Number(repeatRaw) : 1;
|
|
471
360
|
if (repeatRaw !== undefined && (!Number.isFinite(parsedRepeat) || parsedRepeat < 1)) {
|
|
472
|
-
process.stderr.write(
|
|
361
|
+
process.stderr.write(tCli('cli.run.invalid_repeat', lang, { value: repeatRaw }));
|
|
473
362
|
}
|
|
474
363
|
const repeatCount = Math.max(1, Math.floor(parsedRepeat) || 1);
|
|
475
|
-
// --judge-repeat
|
|
364
|
+
// --judge-repeat 同样校验: 非 ≥1 整数时钳到 1
|
|
476
365
|
const judgeRepeatRaw = values['judge-repeat'];
|
|
477
366
|
const parsedJudgeRepeat = judgeRepeatRaw !== undefined ? Number(judgeRepeatRaw) : 1;
|
|
478
367
|
if (judgeRepeatRaw !== undefined && (!Number.isFinite(parsedJudgeRepeat) || parsedJudgeRepeat < 1)) {
|
|
479
|
-
process.stderr.write(
|
|
368
|
+
process.stderr.write(tCli('cli.run.invalid_judge_repeat', lang, { value: judgeRepeatRaw }));
|
|
480
369
|
}
|
|
481
370
|
const judgeRepeatCount = Math.max(1, Math.floor(parsedJudgeRepeat) || 1);
|
|
482
371
|
if (judgeRepeatCount > 1)
|
|
483
372
|
config.judgeRepeat = judgeRepeatCount;
|
|
484
373
|
// --judge-models executor:model,executor:model,... -> JudgeConfig[]
|
|
485
|
-
// 至少 2 个才进 ensemble 模式,1 个等同于 --judge-model
|
|
374
|
+
// 至少 2 个才进 ensemble 模式, 1 个等同于 --judge-model
|
|
486
375
|
const judgeModelsRaw = values['judge-models'];
|
|
487
376
|
if (judgeModelsRaw) {
|
|
488
377
|
const parts = judgeModelsRaw.split(',').map((s) => s.trim()).filter(Boolean);
|
|
@@ -490,7 +379,7 @@ async function handleRun(argv) {
|
|
|
490
379
|
const [executor, ...modelParts] = p.split(':');
|
|
491
380
|
const model = modelParts.join(':');
|
|
492
381
|
if (!executor || !model) {
|
|
493
|
-
throw new Error(
|
|
382
|
+
throw new Error(tCli('cli.run.invalid_judge_models_format', lang, { part: p }));
|
|
494
383
|
}
|
|
495
384
|
return { executor, model };
|
|
496
385
|
});
|
|
@@ -498,8 +387,10 @@ async function handleRun(argv) {
|
|
|
498
387
|
config.judgeModels = judges;
|
|
499
388
|
}
|
|
500
389
|
else if (judges.length === 1) {
|
|
501
|
-
// 单 judge 不走 ensemble
|
|
502
|
-
process.stderr.write(
|
|
390
|
+
// 单 judge 不走 ensemble, 但允许这样写, 等同于 --judge-model + --executor
|
|
391
|
+
process.stderr.write(tCli('cli.run.judge_models_single_warning', lang, {
|
|
392
|
+
executor: judges[0].executor, model: judges[0].model,
|
|
393
|
+
}));
|
|
503
394
|
}
|
|
504
395
|
}
|
|
505
396
|
// --budget-usd / --budget-per-sample-usd / --budget-per-sample-ms:
|
|
@@ -521,7 +412,7 @@ async function handleRun(argv) {
|
|
|
521
412
|
// off so historical reports (judgePromptHash from v2-cot era) can be reproduced.
|
|
522
413
|
if (values['no-debias-length']) {
|
|
523
414
|
config.lengthDebias = false;
|
|
524
|
-
process.stderr.write('
|
|
415
|
+
process.stderr.write(tCli('cli.run.no_debias_length_active', lang));
|
|
525
416
|
}
|
|
526
417
|
// --bootstrap / --bootstrap-samples
|
|
527
418
|
if (values.bootstrap) {
|
|
@@ -529,11 +420,11 @@ async function handleRun(argv) {
|
|
|
529
420
|
const bsRaw = values['bootstrap-samples'];
|
|
530
421
|
const parsedBs = bsRaw !== undefined ? Number(bsRaw) : 1000;
|
|
531
422
|
if (bsRaw !== undefined && (!Number.isFinite(parsedBs) || parsedBs < 100)) {
|
|
532
|
-
process.stderr.write(
|
|
423
|
+
process.stderr.write(tCli('cli.run.invalid_bootstrap_samples', lang, { value: bsRaw }));
|
|
533
424
|
}
|
|
534
425
|
const bsCount = Math.max(100, Math.floor(parsedBs) || 1000);
|
|
535
426
|
if (bsCount > 10000) {
|
|
536
|
-
process.stderr.write(
|
|
427
|
+
process.stderr.write(tCli('cli.run.bootstrap_samples_too_large', lang, { n: bsCount }));
|
|
537
428
|
}
|
|
538
429
|
config.bootstrapSamples = bsCount;
|
|
539
430
|
}
|
|
@@ -545,30 +436,32 @@ async function handleRun(argv) {
|
|
|
545
436
|
repeat: repeatCount,
|
|
546
437
|
onSkillProgress({ phase, skill, current, total }) {
|
|
547
438
|
if (phase === 'start') {
|
|
548
|
-
process.stderr.write(
|
|
439
|
+
process.stderr.write(tCli('cli.run.skill_section', lang, {
|
|
440
|
+
i: current ?? '', n: total ?? '', skill: skill ?? '',
|
|
441
|
+
}));
|
|
549
442
|
}
|
|
550
443
|
},
|
|
551
444
|
});
|
|
552
445
|
console.log(JSON.stringify(report, null, 2));
|
|
553
446
|
if (filePath) {
|
|
554
|
-
process.stderr.write('
|
|
555
|
-
process.stderr.write(
|
|
447
|
+
process.stderr.write(tCli('cli.run.batch_complete', lang));
|
|
448
|
+
process.stderr.write(tCli('cli.run.report_saved', lang, { path: filePath }));
|
|
556
449
|
if (!values['no-serve'] && process.stdout.isTTY) {
|
|
557
450
|
const { createReportServer } = await import('./server/report-server.js');
|
|
558
451
|
const server = createReportServer({ reportsDir: config.outputDir });
|
|
559
452
|
const serverUrl = await server.start();
|
|
560
453
|
const reportUrl = `${serverUrl}/run/${report.id}`;
|
|
561
|
-
process.stderr.write(
|
|
562
|
-
process.stderr.write(
|
|
563
|
-
process.stderr.write('
|
|
454
|
+
process.stderr.write(tCli('cli.run.report_server_running', lang, { url: serverUrl }));
|
|
455
|
+
process.stderr.write(tCli('cli.run.report_server_view', lang, { url: reportUrl }));
|
|
456
|
+
process.stderr.write(tCli('cli.run.report_server_stop', lang));
|
|
564
457
|
const { platform } = await import('node:os');
|
|
565
458
|
const openCmd = platform() === 'darwin' ? 'open' : platform() === 'win32' ? 'start' : 'xdg-open';
|
|
566
459
|
const { execFile: execFileCb } = await import('node:child_process');
|
|
567
460
|
execFileCb(openCmd, [reportUrl], () => { });
|
|
568
461
|
}
|
|
569
462
|
else if (!values['no-serve']) {
|
|
570
|
-
process.stderr.write('
|
|
571
|
-
process.stderr.write(
|
|
463
|
+
process.stderr.write(tCli('cli.run.no_serve_in_non_tty', lang));
|
|
464
|
+
process.stderr.write(tCli('cli.run.no_serve_view_hint', lang, { dir: config.outputDir }));
|
|
572
465
|
}
|
|
573
466
|
}
|
|
574
467
|
return;
|
|
@@ -580,7 +473,7 @@ async function handleRun(argv) {
|
|
|
580
473
|
...config,
|
|
581
474
|
repeat: repeatCount,
|
|
582
475
|
onRepeatProgress({ run, total }) {
|
|
583
|
-
process.stderr.write(
|
|
476
|
+
process.stderr.write(tCli('cli.run.run_section', lang, { i: run, n: total }));
|
|
584
477
|
},
|
|
585
478
|
});
|
|
586
479
|
report = result.report;
|
|
@@ -604,19 +497,22 @@ async function handleRun(argv) {
|
|
|
604
497
|
if (out.result && out.gold) {
|
|
605
498
|
process.stderr.write(formatGoldCompare(out.result, out.gold));
|
|
606
499
|
if (out.result.contaminationWarning) {
|
|
607
|
-
process.stderr.write(
|
|
500
|
+
process.stderr.write(tCli('cli.run.contamination_warning', lang, {
|
|
501
|
+
warning: out.result.contaminationWarning,
|
|
502
|
+
}));
|
|
608
503
|
}
|
|
609
504
|
}
|
|
610
505
|
else {
|
|
611
|
-
process.stderr.write(
|
|
612
|
-
for (const m of out.loadIssues)
|
|
613
|
-
process.stderr.write(
|
|
506
|
+
process.stderr.write(tCli('cli.run.gold_load_failed', lang, { dir: goldDir }));
|
|
507
|
+
for (const m of out.loadIssues) {
|
|
508
|
+
process.stderr.write(tCli('cli.run.gold_load_issue', lang, { message: m }));
|
|
509
|
+
}
|
|
614
510
|
}
|
|
615
511
|
}
|
|
616
512
|
console.log(JSON.stringify(report, null, 2));
|
|
617
513
|
if (filePath) {
|
|
618
|
-
process.stderr.write('
|
|
619
|
-
process.stderr.write(
|
|
514
|
+
process.stderr.write(tCli('cli.run.eval_complete', lang));
|
|
515
|
+
process.stderr.write(tCli('cli.run.report_saved', lang, { path: filePath }));
|
|
620
516
|
if (!values['no-serve'] && process.stdout.isTTY) {
|
|
621
517
|
// Auto-start report server
|
|
622
518
|
const { createReportServer } = await import('./server/report-server.js');
|
|
@@ -625,9 +521,9 @@ async function handleRun(argv) {
|
|
|
625
521
|
});
|
|
626
522
|
const serverUrl = await server.start();
|
|
627
523
|
const reportUrl = `${serverUrl}/run/${report.id}`;
|
|
628
|
-
process.stderr.write(
|
|
629
|
-
process.stderr.write(
|
|
630
|
-
process.stderr.write('
|
|
524
|
+
process.stderr.write(tCli('cli.run.report_server_running', lang, { url: serverUrl }));
|
|
525
|
+
process.stderr.write(tCli('cli.run.report_server_view', lang, { url: reportUrl }));
|
|
526
|
+
process.stderr.write(tCli('cli.run.report_server_stop', lang));
|
|
631
527
|
// Auto-open report in browser
|
|
632
528
|
const { platform } = await import('node:os');
|
|
633
529
|
const openCmd = platform() === 'darwin' ? 'open' : platform() === 'win32' ? 'start' : 'xdg-open';
|
|
@@ -635,13 +531,13 @@ async function handleRun(argv) {
|
|
|
635
531
|
execFileCb(openCmd, [reportUrl], () => { });
|
|
636
532
|
}
|
|
637
533
|
else if (!values['no-serve']) {
|
|
638
|
-
process.stderr.write('
|
|
639
|
-
process.stderr.write(
|
|
534
|
+
process.stderr.write(tCli('cli.run.no_serve_in_non_tty', lang));
|
|
535
|
+
process.stderr.write(tCli('cli.run.no_serve_view_hint', lang, { dir: config.outputDir }));
|
|
640
536
|
}
|
|
641
537
|
}
|
|
642
538
|
}
|
|
643
539
|
catch (err) {
|
|
644
|
-
console.error(
|
|
540
|
+
console.error(tCli('cli.common.error_prefix', lang, { message: err.message }));
|
|
645
541
|
process.exit(1);
|
|
646
542
|
}
|
|
647
543
|
}
|
|
@@ -649,9 +545,11 @@ async function handleRun(argv) {
|
|
|
649
545
|
// handleReport
|
|
650
546
|
// ---------------------------------------------------------------------------
|
|
651
547
|
async function handleReport(argv) {
|
|
548
|
+
const lang = langFromArgv(argv);
|
|
652
549
|
const { values } = parseArgs({
|
|
653
550
|
args: argv,
|
|
654
551
|
options: {
|
|
552
|
+
...COMMON_OPTIONS,
|
|
655
553
|
port: { type: 'string', default: '7799' },
|
|
656
554
|
'reports-dir': { type: 'string', default: DEFAULT_REPORTS_DIR },
|
|
657
555
|
export: { type: 'string' },
|
|
@@ -684,7 +582,7 @@ async function handleReport(argv) {
|
|
|
684
582
|
const store = createFileStore(resolve(values['reports-dir']));
|
|
685
583
|
const report = await store.get(values.export);
|
|
686
584
|
if (!report) {
|
|
687
|
-
console.error(
|
|
585
|
+
console.error(tCli('cli.common.report_not_found', lang, { id: values.export }));
|
|
688
586
|
process.exit(1);
|
|
689
587
|
}
|
|
690
588
|
const html = report.each ? renderEachRunDetail(report) : renderRunDetail(report);
|
|
@@ -779,10 +677,12 @@ function parseLastWindow(spec) {
|
|
|
779
677
|
return new Date(Date.now() - ms).toISOString();
|
|
780
678
|
}
|
|
781
679
|
async function handleAnalyze(argv) {
|
|
680
|
+
const lang = langFromArgv(argv);
|
|
782
681
|
const { values, positionals } = parseArgs({
|
|
783
682
|
args: argv,
|
|
784
683
|
allowPositionals: true,
|
|
785
684
|
options: {
|
|
685
|
+
...COMMON_OPTIONS,
|
|
786
686
|
kb: { type: 'string' },
|
|
787
687
|
last: { type: 'string' },
|
|
788
688
|
from: { type: 'string' },
|
|
@@ -793,7 +693,7 @@ async function handleAnalyze(argv) {
|
|
|
793
693
|
});
|
|
794
694
|
const dir = positionals[0];
|
|
795
695
|
if (!dir) {
|
|
796
|
-
console.error('
|
|
696
|
+
console.error(tCli('cli.help.analyze_usage', lang));
|
|
797
697
|
process.exit(1);
|
|
798
698
|
}
|
|
799
699
|
const tracePath = resolve(dir);
|
|
@@ -842,29 +742,32 @@ async function handleAnalyze(argv) {
|
|
|
842
742
|
console.log(skillRows.join('\n'));
|
|
843
743
|
console.log('');
|
|
844
744
|
console.log(`report written to: ${jsonPath}`);
|
|
845
|
-
console.log(
|
|
745
|
+
console.log(tCli('cli.analyze.view_in_browser', lang));
|
|
846
746
|
}
|
|
847
747
|
async function handleInit(argv) {
|
|
748
|
+
const lang = langFromArgv(argv);
|
|
848
749
|
const targetDir = resolve(argv[0] || '.');
|
|
849
750
|
const { writeFileSync, mkdirSync } = await import('node:fs');
|
|
850
751
|
mkdirSync(join(targetDir, 'skills'), { recursive: true });
|
|
851
752
|
writeFileSync(join(targetDir, 'eval-samples.json'), INIT_SAMPLES);
|
|
852
753
|
writeFileSync(join(targetDir, 'skills', 'v1.md'), INIT_SKILL_V1);
|
|
853
754
|
writeFileSync(join(targetDir, 'skills', 'v2.md'), INIT_SKILL_V2);
|
|
854
|
-
console.log(
|
|
755
|
+
console.log(tCli('cli.init.scaffolded', lang, { dir: targetDir }));
|
|
855
756
|
console.log('');
|
|
856
|
-
console.log('
|
|
857
|
-
console.log('
|
|
858
|
-
console.log('
|
|
859
|
-
console.log('
|
|
757
|
+
console.log(tCli('cli.init.next_steps_title', lang));
|
|
758
|
+
console.log(tCli('cli.init.next_step_edit_samples', lang));
|
|
759
|
+
console.log(tCli('cli.init.next_step_edit_skills', lang));
|
|
760
|
+
console.log(tCli('cli.init.next_step_run', lang));
|
|
860
761
|
}
|
|
861
762
|
// ---------------------------------------------------------------------------
|
|
862
763
|
// handleGenSamples
|
|
863
764
|
// ---------------------------------------------------------------------------
|
|
864
765
|
async function handleGenSamples(argv) {
|
|
766
|
+
const lang = langFromArgv(argv);
|
|
865
767
|
const { values } = parseArgs({
|
|
866
768
|
args: argv,
|
|
867
769
|
options: {
|
|
770
|
+
...COMMON_OPTIONS,
|
|
868
771
|
each: { type: 'boolean', default: false },
|
|
869
772
|
count: { type: 'string', default: '5' },
|
|
870
773
|
model: { type: 'string', default: 'sonnet' },
|
|
@@ -881,7 +784,7 @@ async function handleGenSamples(argv) {
|
|
|
881
784
|
// Batch mode: generate for all skills missing eval-samples
|
|
882
785
|
const skillDir = resolve(values['skill-dir']);
|
|
883
786
|
if (!existsSync(skillDir)) {
|
|
884
|
-
console.error(
|
|
787
|
+
console.error(tCli('cli.common.skill_dir_not_found', lang, { path: skillDir }));
|
|
885
788
|
process.exit(1);
|
|
886
789
|
}
|
|
887
790
|
const { readdirSync, statSync } = await import('node:fs');
|
|
@@ -909,55 +812,63 @@ async function handleGenSamples(argv) {
|
|
|
909
812
|
continue;
|
|
910
813
|
}
|
|
911
814
|
if (existsSync(samplesPath)) {
|
|
912
|
-
process.stderr.write(
|
|
815
|
+
process.stderr.write(tCli('cli.gen.skill_skipped_existing', lang, { name }));
|
|
913
816
|
continue;
|
|
914
817
|
}
|
|
915
|
-
process.stderr.write(
|
|
818
|
+
process.stderr.write(tCli('cli.gen.skill_generating', lang, { name, count }));
|
|
916
819
|
try {
|
|
917
820
|
const skillContent = readFileSync(skillPath, 'utf-8');
|
|
918
821
|
const { samples, costUSD } = await generateSamples({ skillContent, count, model });
|
|
919
822
|
writeFileSync(samplesPath, JSON.stringify(samples, null, 2));
|
|
920
|
-
|
|
823
|
+
const cost = costUSD > 0 ? ` $${costUSD.toFixed(4)}` : '';
|
|
824
|
+
process.stderr.write(tCli('cli.gen.skill_done', lang, {
|
|
825
|
+
name, n: samples.length, path: samplesPath, cost,
|
|
826
|
+
}));
|
|
921
827
|
generated++;
|
|
922
828
|
}
|
|
923
829
|
catch (err) {
|
|
924
|
-
process.stderr.write(
|
|
830
|
+
process.stderr.write(tCli('cli.gen.skill_failed', lang, {
|
|
831
|
+
name, message: err.message,
|
|
832
|
+
}));
|
|
925
833
|
}
|
|
926
834
|
}
|
|
927
835
|
if (generated === 0) {
|
|
928
|
-
console.log('
|
|
836
|
+
console.log(tCli('cli.gen.batch_none_needed', lang));
|
|
929
837
|
}
|
|
930
838
|
else {
|
|
931
|
-
console.log(
|
|
839
|
+
console.log(tCli('cli.gen.batch_summary', lang, { n: generated }));
|
|
932
840
|
}
|
|
933
841
|
}
|
|
934
842
|
else {
|
|
935
843
|
// Single skill mode
|
|
936
844
|
const skillPath = argv.find((a) => !a.startsWith('-'));
|
|
937
845
|
if (!skillPath) {
|
|
938
|
-
console.error('
|
|
846
|
+
console.error(tCli('cli.gen.specify_skill_path', lang));
|
|
939
847
|
process.exit(1);
|
|
940
848
|
}
|
|
941
849
|
const resolvedPath = resolve(skillPath);
|
|
942
850
|
if (!existsSync(resolvedPath)) {
|
|
943
|
-
console.error(
|
|
851
|
+
console.error(tCli('cli.common.skill_file_not_found', lang, { path: resolvedPath }));
|
|
944
852
|
process.exit(1);
|
|
945
853
|
}
|
|
946
854
|
const skillContent = readFileSync(resolvedPath, 'utf-8');
|
|
947
855
|
const outputPath = resolve('eval-samples.json');
|
|
948
856
|
if (existsSync(outputPath)) {
|
|
949
|
-
console.error(
|
|
857
|
+
console.error(tCli('cli.gen.samples_already_exists', lang));
|
|
950
858
|
process.exit(1);
|
|
951
859
|
}
|
|
952
|
-
process.stderr.write(
|
|
860
|
+
process.stderr.write(tCli('cli.gen.single_generating', lang, { count }));
|
|
953
861
|
try {
|
|
954
862
|
const { samples, costUSD } = await generateSamples({ skillContent, count, model });
|
|
955
863
|
writeFileSync(outputPath, JSON.stringify(samples, null, 2));
|
|
956
|
-
|
|
957
|
-
|
|
864
|
+
const cost = costUSD > 0 ? ` $${costUSD.toFixed(4)}` : '';
|
|
865
|
+
process.stderr.write(tCli('cli.gen.single_done', lang, {
|
|
866
|
+
n: samples.length, path: outputPath, cost,
|
|
867
|
+
}));
|
|
868
|
+
console.log(tCli('cli.gen.review_hint', lang));
|
|
958
869
|
}
|
|
959
870
|
catch (err) {
|
|
960
|
-
console.error(
|
|
871
|
+
console.error(tCli('cli.gen.failed', lang, { message: err.message }));
|
|
961
872
|
process.exit(1);
|
|
962
873
|
}
|
|
963
874
|
}
|
|
@@ -966,9 +877,11 @@ async function handleGenSamples(argv) {
|
|
|
966
877
|
// handleEvolve
|
|
967
878
|
// ---------------------------------------------------------------------------
|
|
968
879
|
async function handleEvolve(argv) {
|
|
880
|
+
const lang = langFromArgv(argv);
|
|
969
881
|
const { values } = parseArgs({
|
|
970
882
|
args: argv,
|
|
971
883
|
options: {
|
|
884
|
+
...COMMON_OPTIONS,
|
|
972
885
|
rounds: { type: 'string', default: '5' },
|
|
973
886
|
target: { type: 'string' },
|
|
974
887
|
samples: { type: 'string', default: 'eval-samples.json' },
|
|
@@ -985,7 +898,7 @@ async function handleEvolve(argv) {
|
|
|
985
898
|
});
|
|
986
899
|
const skillPath = argv.find((a) => !a.startsWith('-'));
|
|
987
900
|
if (!skillPath) {
|
|
988
|
-
console.error('
|
|
901
|
+
console.error(tCli('cli.evolve.specify_skill_path', lang));
|
|
989
902
|
process.exit(1);
|
|
990
903
|
}
|
|
991
904
|
let samplesFile = values.samples ?? 'eval-samples.json';
|
|
@@ -996,7 +909,7 @@ async function handleEvolve(argv) {
|
|
|
996
909
|
samplesFile = 'eval-samples.yml';
|
|
997
910
|
}
|
|
998
911
|
const { evolveSkill } = await import('./authoring/evolver.js');
|
|
999
|
-
process.stderr.write(
|
|
912
|
+
process.stderr.write(tCli('cli.evolve.section_header', lang, { path: skillPath }));
|
|
1000
913
|
try {
|
|
1001
914
|
const result = await evolveSkill({
|
|
1002
915
|
skillPath: resolve(skillPath),
|
|
@@ -1010,59 +923,93 @@ async function handleEvolve(argv) {
|
|
|
1010
923
|
concurrency: Math.max(1, Number(values.concurrency) || 1),
|
|
1011
924
|
timeoutMs: Math.max(1, Number(values.timeout) || 120) * 1000,
|
|
1012
925
|
skipPreflight: values['skip-preflight'],
|
|
1013
|
-
onProgress:
|
|
926
|
+
onProgress: makeOnProgress(lang),
|
|
1014
927
|
onRoundProgress({ round, totalRounds: _totalRounds, phase, score, delta, accepted, costUSD, error }) {
|
|
1015
928
|
if (phase === 'baseline') {
|
|
1016
|
-
process.stderr.write(
|
|
929
|
+
process.stderr.write(tCli('cli.evolve.round_baseline', lang, {
|
|
930
|
+
score: score.toFixed(2), cost: costUSD.toFixed(4),
|
|
931
|
+
}));
|
|
1017
932
|
}
|
|
1018
933
|
else if (phase === 'error') {
|
|
1019
|
-
process.stderr.write(
|
|
934
|
+
process.stderr.write(tCli('cli.evolve.round_error', lang, {
|
|
935
|
+
round, error: String(error ?? ''),
|
|
936
|
+
}));
|
|
1020
937
|
}
|
|
1021
938
|
else if (phase === 'done') {
|
|
1022
|
-
const
|
|
939
|
+
const delta_ = delta >= 0 ? `+${delta.toFixed(2)}` : delta.toFixed(2);
|
|
1023
940
|
const status = accepted ? '✓ ACCEPT' : '✗ REJECT';
|
|
1024
|
-
process.stderr.write(
|
|
941
|
+
process.stderr.write(tCli('cli.evolve.round_done', lang, {
|
|
942
|
+
round, score: score.toFixed(2), delta: delta_, status, cost: costUSD.toFixed(4),
|
|
943
|
+
}));
|
|
1025
944
|
}
|
|
1026
945
|
},
|
|
1027
946
|
});
|
|
1028
947
|
const improvement = result.startScore > 0
|
|
1029
948
|
? ((result.finalScore - result.startScore) / result.startScore * 100).toFixed(1)
|
|
1030
949
|
: '0';
|
|
1031
|
-
process.stderr.write(
|
|
1032
|
-
|
|
1033
|
-
|
|
950
|
+
process.stderr.write(tCli('cli.evolve.summary', lang, {
|
|
951
|
+
start: result.startScore.toFixed(2), final: result.finalScore.toFixed(2),
|
|
952
|
+
percent: improvement, rounds: result.totalRounds, cost: result.totalCostUSD.toFixed(4),
|
|
953
|
+
}));
|
|
954
|
+
process.stderr.write(tCli('cli.evolve.best_path', lang, {
|
|
955
|
+
best: result.bestSkillPath, target: resolve(skillPath),
|
|
956
|
+
}));
|
|
957
|
+
process.stderr.write(tCli('cli.evolve.versions_saved', lang, {
|
|
958
|
+
dir: join(resolve(skillPath, '..'), 'evolve'),
|
|
959
|
+
}));
|
|
1034
960
|
if (result.reportId) {
|
|
1035
|
-
process.stderr.write(
|
|
961
|
+
process.stderr.write(tCli('cli.evolve.report_link', lang, { id: result.reportId }));
|
|
1036
962
|
}
|
|
1037
963
|
console.log(JSON.stringify(result, null, 2));
|
|
1038
964
|
}
|
|
1039
965
|
catch (err) {
|
|
1040
|
-
console.error(
|
|
966
|
+
console.error(tCli('cli.common.error_prefix', lang, { message: err.message }));
|
|
1041
967
|
process.exit(1);
|
|
1042
968
|
}
|
|
1043
969
|
}
|
|
1044
970
|
// ---------------------------------------------------------------------------
|
|
1045
|
-
//
|
|
971
|
+
// handleGate — 跑评测 + 应用 gate, exit code 0/1 适合 CI/CD pipeline 调用。
|
|
972
|
+
// 内部 = runEvaluation + computeVerdict + formatVerdictText, 与 bench verdict
|
|
973
|
+
// 共用决策内核(只是 verdict 读已有报告, gate 跑完再判)。
|
|
1046
974
|
// ---------------------------------------------------------------------------
|
|
1047
|
-
async function
|
|
975
|
+
async function handleGate(argv) {
|
|
976
|
+
const lang = langFromArgv(argv);
|
|
977
|
+
if (argv[0] === '--help' || argv[0] === '-h') {
|
|
978
|
+
console.log(tCli('cli.help.main', lang).trim());
|
|
979
|
+
process.exit(0);
|
|
980
|
+
}
|
|
1048
981
|
const { values, config } = parseRunConfig(argv, {
|
|
1049
982
|
threshold: { type: 'string', default: '3.5' },
|
|
983
|
+
'trivial-diff': { type: 'string' },
|
|
1050
984
|
});
|
|
1051
985
|
const { runEvaluation } = await import('./eval-workflows/run-evaluation.js');
|
|
1052
|
-
config.onProgress =
|
|
986
|
+
config.onProgress = makeOnProgress(lang);
|
|
1053
987
|
try {
|
|
1054
988
|
const { report } = (await runEvaluation(config));
|
|
1055
|
-
const threshold = Number(values.threshold);
|
|
1056
989
|
if (report.dryRun) {
|
|
1057
|
-
console.log('
|
|
990
|
+
console.log('Gate dry-run: no scores to check');
|
|
1058
991
|
process.exit(0);
|
|
1059
992
|
}
|
|
1060
|
-
//
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
993
|
+
// gate 内核 = run + verdict, 自动覆盖 omk 全部决策维度(三层 layer-gate /
|
|
994
|
+
// bootstrap diff CI / saturation / Krippendorff α)。computeVerdict 是单一
|
|
995
|
+
// 决策源, exit code 跟 verdict.level 走 — 数据 underpowered 直接 FAIL,
|
|
996
|
+
// 堵住"过 PASS 就 deploy"的漏洞。
|
|
997
|
+
const { computeVerdict, formatVerdictText } = await import('./eval-core/verdict.js');
|
|
998
|
+
const result = computeVerdict(report, {
|
|
999
|
+
gateThreshold: Number(values.threshold),
|
|
1000
|
+
triviallySmallDiff: values['trivial-diff'] != null ? Number(values['trivial-diff']) : undefined,
|
|
1001
|
+
});
|
|
1002
|
+
console.log(formatVerdictText(result, { verbose: true }));
|
|
1003
|
+
// exit code 与 handleVerdict 对齐:只有 PROGRESS / SOLO-pass 才 0,
|
|
1004
|
+
// NOISE / UNDERPOWERED / CAUTIOUS / REGRESS 全 1。pipeline `omk bench gate
|
|
1005
|
+
// && deploy` 数据不显著就不会误 deploy。
|
|
1006
|
+
if (result.level === 'PROGRESS') {
|
|
1007
|
+
process.exit(0);
|
|
1008
|
+
}
|
|
1009
|
+
if (result.level === 'SOLO' && result.headline.includes('PASS')) {
|
|
1010
|
+
process.exit(0);
|
|
1011
|
+
}
|
|
1012
|
+
process.exit(1);
|
|
1066
1013
|
}
|
|
1067
1014
|
catch (err) {
|
|
1068
1015
|
console.error(`Error: ${err.message}`);
|
|
@@ -1073,6 +1020,7 @@ async function handleCi(argv) {
|
|
|
1073
1020
|
// handleDiff
|
|
1074
1021
|
// ---------------------------------------------------------------------------
|
|
1075
1022
|
async function handleDiff(argv) {
|
|
1023
|
+
const lang = langFromArgv(argv);
|
|
1076
1024
|
// Flag-aware split: separate positional report IDs from flags so we can support
|
|
1077
1025
|
// omk bench diff <id> — within-report sample-level (v0.22)
|
|
1078
1026
|
// omk bench diff <id1> <id2> — cross-report variant-level (legacy)
|
|
@@ -1094,22 +1042,13 @@ async function handleDiff(argv) {
|
|
|
1094
1042
|
}
|
|
1095
1043
|
}
|
|
1096
1044
|
if (positional.length === 0) {
|
|
1097
|
-
console.error(
|
|
1098
|
-
'Usage:',
|
|
1099
|
-
' omk bench diff <reportId> within-report per-sample diff (v0.22)',
|
|
1100
|
-
' omk bench diff <reportId1> <reportId2> cross-report variant-level diff',
|
|
1101
|
-
'',
|
|
1102
|
-
'Options:',
|
|
1103
|
-
' --regressions-only 只列 treatment < control 的样本',
|
|
1104
|
-
' --threshold <num> regression 阈值 (default 0,即任一负 Δ 算回退)',
|
|
1105
|
-
' --variant <name> within-report 模式下指定要钻取的 variant (default: variants[1])',
|
|
1106
|
-
' --top <n> 只列差距最大的前 N 个样本',
|
|
1107
|
-
].join('\n'));
|
|
1045
|
+
console.error(tCli('cli.help.diff_usage', lang));
|
|
1108
1046
|
process.exit(positional.length === 0 ? 1 : 0);
|
|
1109
1047
|
}
|
|
1110
1048
|
const { values } = parseArgs({
|
|
1111
1049
|
args: flagArgs,
|
|
1112
1050
|
options: {
|
|
1051
|
+
...COMMON_OPTIONS,
|
|
1113
1052
|
'regressions-only': { type: 'boolean', default: false },
|
|
1114
1053
|
threshold: { type: 'string' },
|
|
1115
1054
|
variant: { type: 'string' },
|
|
@@ -1120,18 +1059,18 @@ async function handleDiff(argv) {
|
|
|
1120
1059
|
const { createFileStore } = await import('./server/report-store.js');
|
|
1121
1060
|
const store = createFileStore(resolve(DEFAULT_REPORTS_DIR));
|
|
1122
1061
|
if (positional.length === 1) {
|
|
1123
|
-
await runSampleLevelDiff(positional[0], store, values);
|
|
1062
|
+
await runSampleLevelDiff(positional[0], store, values, lang);
|
|
1124
1063
|
return;
|
|
1125
1064
|
}
|
|
1126
1065
|
const [id1, id2] = positional;
|
|
1127
1066
|
const r1 = await store.get(id1);
|
|
1128
1067
|
const r2 = await store.get(id2);
|
|
1129
1068
|
if (!r1) {
|
|
1130
|
-
console.error(
|
|
1069
|
+
console.error(tCli('cli.common.report_not_found', lang, { id: id1 }));
|
|
1131
1070
|
process.exit(1);
|
|
1132
1071
|
}
|
|
1133
1072
|
if (!r2) {
|
|
1134
|
-
console.error(
|
|
1073
|
+
console.error(tCli('cli.common.report_not_found', lang, { id: id2 }));
|
|
1135
1074
|
process.exit(1);
|
|
1136
1075
|
}
|
|
1137
1076
|
console.log(`\n Diff: ${id1} → ${id2}\n`);
|
|
@@ -1187,10 +1126,10 @@ async function handleDiff(argv) {
|
|
|
1187
1126
|
* Default focus is variants[0] (control) vs variants[1] (treatment), but
|
|
1188
1127
|
* `--variant` overrides which variant is the "treatment" side.
|
|
1189
1128
|
*/
|
|
1190
|
-
async function runSampleLevelDiff(reportId, store, flags) {
|
|
1129
|
+
async function runSampleLevelDiff(reportId, store, flags, lang) {
|
|
1191
1130
|
const report = await store.get(reportId);
|
|
1192
1131
|
if (!report) {
|
|
1193
|
-
console.error(
|
|
1132
|
+
console.error(tCli('cli.common.report_not_found', lang, { id: reportId }));
|
|
1194
1133
|
process.exit(1);
|
|
1195
1134
|
}
|
|
1196
1135
|
const variants = report.meta?.variants ?? [];
|
|
@@ -1267,27 +1206,18 @@ async function runSampleLevelDiff(reportId, store, flags) {
|
|
|
1267
1206
|
// handleGold — gold dataset workflow (init / validate / compare)
|
|
1268
1207
|
// ---------------------------------------------------------------------------
|
|
1269
1208
|
async function handleGold(argv) {
|
|
1209
|
+
const lang = langFromArgv(argv);
|
|
1270
1210
|
const sub = argv[0];
|
|
1271
1211
|
const rest = argv.slice(1);
|
|
1272
1212
|
if (!sub || sub === '--help' || sub === '-h') {
|
|
1273
|
-
console.log(
|
|
1274
|
-
'',
|
|
1275
|
-
'Usage: omk bench gold <subcommand>',
|
|
1276
|
-
'',
|
|
1277
|
-
'Subcommands:',
|
|
1278
|
-
' init [--out <dir>] [--annotator <id>] 生成空白 gold dataset 模板',
|
|
1279
|
-
' validate <dir> 校验数据集结构',
|
|
1280
|
-
' compare <reportId> --gold-dir <dir> 与已有 report 计算 α/κ/Pearson',
|
|
1281
|
-
' [--variant <name>] [--reports-dir <d>]',
|
|
1282
|
-
' [--bootstrap-samples N] [--seed N]',
|
|
1283
|
-
'',
|
|
1284
|
-
].join('\n'));
|
|
1213
|
+
console.log(tCli('cli.help.gold', lang));
|
|
1285
1214
|
process.exit(sub ? 0 : 1);
|
|
1286
1215
|
}
|
|
1287
1216
|
if (sub === 'init') {
|
|
1288
1217
|
const { values } = parseArgs({
|
|
1289
1218
|
args: rest,
|
|
1290
1219
|
options: {
|
|
1220
|
+
...COMMON_OPTIONS,
|
|
1291
1221
|
out: { type: 'string', default: './gold-dataset' },
|
|
1292
1222
|
annotator: { type: 'string' },
|
|
1293
1223
|
},
|
|
@@ -1298,10 +1228,12 @@ async function handleGold(argv) {
|
|
|
1298
1228
|
const written = initGoldDataset(values.out, {
|
|
1299
1229
|
annotator: values.annotator,
|
|
1300
1230
|
});
|
|
1301
|
-
console.log(
|
|
1231
|
+
console.log(tCli('cli.gold.created_files', lang, {
|
|
1232
|
+
n: written.length, dir: values.out,
|
|
1233
|
+
}));
|
|
1302
1234
|
for (const p of written)
|
|
1303
1235
|
console.log(` ${p}`);
|
|
1304
|
-
console.log('
|
|
1236
|
+
console.log(tCli('cli.gold.next_step_edit_annotations', lang));
|
|
1305
1237
|
}
|
|
1306
1238
|
catch (err) {
|
|
1307
1239
|
console.error(err.message);
|
|
@@ -1312,13 +1244,13 @@ async function handleGold(argv) {
|
|
|
1312
1244
|
if (sub === 'validate') {
|
|
1313
1245
|
const dir = rest[0];
|
|
1314
1246
|
if (!dir) {
|
|
1315
|
-
console.error('
|
|
1247
|
+
console.error(tCli('cli.common.usage_gold_validate', lang));
|
|
1316
1248
|
process.exit(1);
|
|
1317
1249
|
}
|
|
1318
1250
|
const { validateGoldDataset } = await import('./grading/gold-cli.js');
|
|
1319
1251
|
const result = validateGoldDataset(dir);
|
|
1320
1252
|
if (result.ok) {
|
|
1321
|
-
console.log(
|
|
1253
|
+
console.log(tCli('cli.gold.validate_ok', lang, { n: result.sampleCount }));
|
|
1322
1254
|
return;
|
|
1323
1255
|
}
|
|
1324
1256
|
console.error(`✗ gold dataset has ${result.issues.length} issue(s):`);
|
|
@@ -1335,6 +1267,7 @@ async function handleGold(argv) {
|
|
|
1335
1267
|
const { values } = parseArgs({
|
|
1336
1268
|
args: rest.slice(1),
|
|
1337
1269
|
options: {
|
|
1270
|
+
...COMMON_OPTIONS,
|
|
1338
1271
|
'gold-dir': { type: 'string' },
|
|
1339
1272
|
variant: { type: 'string' },
|
|
1340
1273
|
'reports-dir': { type: 'string', default: DEFAULT_REPORTS_DIR },
|
|
@@ -1366,7 +1299,7 @@ async function handleGold(argv) {
|
|
|
1366
1299
|
const store = createFileStore(resolve(values['reports-dir']));
|
|
1367
1300
|
const report = await store.get(reportId);
|
|
1368
1301
|
if (!report) {
|
|
1369
|
-
console.error(
|
|
1302
|
+
console.error(tCli('cli.common.report_not_found', lang, { id: reportId }));
|
|
1370
1303
|
process.exit(1);
|
|
1371
1304
|
}
|
|
1372
1305
|
const samples = Math.max(100, Number(values['bootstrap-samples']) || 1000);
|
|
@@ -1388,27 +1321,11 @@ async function handleGold(argv) {
|
|
|
1388
1321
|
// handleDebiasValidate — measure length-debias prompt sensitivity (Phase 3a)
|
|
1389
1322
|
// ---------------------------------------------------------------------------
|
|
1390
1323
|
async function handleDebiasValidate(argv) {
|
|
1324
|
+
const lang = langFromArgv(argv);
|
|
1391
1325
|
const sub = argv[0];
|
|
1392
1326
|
const rest = argv.slice(1);
|
|
1393
1327
|
if (!sub || sub === '--help' || sub === '-h') {
|
|
1394
|
-
console.log(
|
|
1395
|
-
'',
|
|
1396
|
-
'Usage: omk bench debias-validate <kind> <reportId> [options]',
|
|
1397
|
-
'',
|
|
1398
|
-
'Kinds:',
|
|
1399
|
-
' length re-judge with the opposite length-debias setting and bootstrap CI',
|
|
1400
|
-
' on the score diff. Cost ~doubles vs the original judge pass.',
|
|
1401
|
-
'',
|
|
1402
|
-
'Options:',
|
|
1403
|
-
' --reports-dir <dir> report store dir (default: ~/.oh-my-knowledge/reports)',
|
|
1404
|
-
' --samples <path> override samples file (default: from report.meta.request)',
|
|
1405
|
-
' --variant <name> which variant to validate (default: first)',
|
|
1406
|
-
' --judge-executor <name> executor for judge calls (default: claude)',
|
|
1407
|
-
' --judge-model <model> judge model id (default: from report)',
|
|
1408
|
-
' --bootstrap-samples N bootstrap iterations (default 1000)',
|
|
1409
|
-
' --seed N deterministic CI seed',
|
|
1410
|
-
'',
|
|
1411
|
-
].join('\n'));
|
|
1328
|
+
console.log(tCli('cli.help.debias_validate', lang));
|
|
1412
1329
|
process.exit(sub ? 0 : 1);
|
|
1413
1330
|
}
|
|
1414
1331
|
if (sub !== 'length') {
|
|
@@ -1423,6 +1340,7 @@ async function handleDebiasValidate(argv) {
|
|
|
1423
1340
|
const { values } = parseArgs({
|
|
1424
1341
|
args: rest.slice(1),
|
|
1425
1342
|
options: {
|
|
1343
|
+
...COMMON_OPTIONS,
|
|
1426
1344
|
'reports-dir': { type: 'string', default: DEFAULT_REPORTS_DIR },
|
|
1427
1345
|
samples: { type: 'string' },
|
|
1428
1346
|
variant: { type: 'string' },
|
|
@@ -1437,7 +1355,7 @@ async function handleDebiasValidate(argv) {
|
|
|
1437
1355
|
const store = createFileStore(resolve(values['reports-dir']));
|
|
1438
1356
|
const report = await store.get(reportId);
|
|
1439
1357
|
if (!report) {
|
|
1440
|
-
console.error(
|
|
1358
|
+
console.error(tCli('cli.common.report_not_found', lang, { id: reportId }));
|
|
1441
1359
|
process.exit(1);
|
|
1442
1360
|
}
|
|
1443
1361
|
// Resolve samples path: --samples overrides; otherwise read from report.meta.request.
|
|
@@ -1452,10 +1370,10 @@ async function handleDebiasValidate(argv) {
|
|
|
1452
1370
|
const judgeModel = values['judge-model']
|
|
1453
1371
|
?? report.meta?.judgeModel;
|
|
1454
1372
|
if (!judgeModel) {
|
|
1455
|
-
console.error('
|
|
1373
|
+
console.error(tCli('cli.common.no_judge_model', lang));
|
|
1456
1374
|
process.exit(1);
|
|
1457
1375
|
}
|
|
1458
|
-
process.stderr.write('
|
|
1376
|
+
process.stderr.write(tCli('cli.debias.warn_cost_doubles', lang));
|
|
1459
1377
|
const { createExecutor } = await import('./executors/index.js');
|
|
1460
1378
|
const judgeExecutor = createExecutor(values['judge-executor']);
|
|
1461
1379
|
const { validateLengthDebias, formatDebiasValidate } = await import('./grading/debias-validate.js');
|
|
@@ -1479,29 +1397,16 @@ async function handleDebiasValidate(argv) {
|
|
|
1479
1397
|
// handleSaturation — re-compute saturation verdict from a finished report
|
|
1480
1398
|
// ---------------------------------------------------------------------------
|
|
1481
1399
|
async function handleSaturation(argv) {
|
|
1400
|
+
const lang = langFromArgv(argv);
|
|
1482
1401
|
const reportId = argv[0];
|
|
1483
1402
|
if (!reportId || reportId === '--help' || reportId === '-h') {
|
|
1484
|
-
console.log(
|
|
1485
|
-
'',
|
|
1486
|
-
'Usage: omk bench saturation <reportId> [options]',
|
|
1487
|
-
'',
|
|
1488
|
-
'回答"我跑够样本了吗?"。复述已有 report 中持久化的饱和判定。',
|
|
1489
|
-
'',
|
|
1490
|
-
'注:本命令读取 run 时跑出的 verdict(运行时已用 method=bootstrap-ci-width',
|
|
1491
|
-
'默认阈值 + 3 窗口 持续条件)。如要换 method/threshold 重新计算,需要重跑',
|
|
1492
|
-
'`omk bench run --repeat ≥ 5`(运行时持久化的 trace 不含原始分数,无法',
|
|
1493
|
-
'在事后用其他参数复算)。',
|
|
1494
|
-
'',
|
|
1495
|
-
'Options:',
|
|
1496
|
-
' --reports-dir <dir> report store dir (default: ~/.oh-my-knowledge/reports)',
|
|
1497
|
-
' --variant <name> 只看一个 variant (default: all)',
|
|
1498
|
-
'',
|
|
1499
|
-
].join('\n'));
|
|
1403
|
+
console.log(tCli('cli.help.saturation', lang));
|
|
1500
1404
|
process.exit(reportId ? 0 : 1);
|
|
1501
1405
|
}
|
|
1502
1406
|
const { values } = parseArgs({
|
|
1503
1407
|
args: argv.slice(1),
|
|
1504
1408
|
options: {
|
|
1409
|
+
...COMMON_OPTIONS,
|
|
1505
1410
|
'reports-dir': { type: 'string', default: DEFAULT_REPORTS_DIR },
|
|
1506
1411
|
variant: { type: 'string' },
|
|
1507
1412
|
},
|
|
@@ -1511,12 +1416,12 @@ async function handleSaturation(argv) {
|
|
|
1511
1416
|
const store = createFileStore(resolve(values['reports-dir']));
|
|
1512
1417
|
const report = await store.get(reportId);
|
|
1513
1418
|
if (!report) {
|
|
1514
|
-
console.error(
|
|
1419
|
+
console.error(tCli('cli.common.report_not_found', lang, { id: reportId }));
|
|
1515
1420
|
process.exit(1);
|
|
1516
1421
|
}
|
|
1517
1422
|
const saturation = report.variance?.saturation;
|
|
1518
1423
|
if (!saturation) {
|
|
1519
|
-
console.error('
|
|
1424
|
+
console.error(tCli('cli.saturation.no_data', lang));
|
|
1520
1425
|
process.exit(1);
|
|
1521
1426
|
}
|
|
1522
1427
|
// Print the persisted verdict from the original run. The trace stores
|
|
@@ -1527,22 +1432,32 @@ async function handleSaturation(argv) {
|
|
|
1527
1432
|
// run time to enable post-hoc parameter sweeps.
|
|
1528
1433
|
const variants = report.meta.variants ?? [];
|
|
1529
1434
|
const targetVariants = values.variant ? [values.variant] : variants;
|
|
1530
|
-
console.log(
|
|
1435
|
+
console.log(tCli('cli.saturation.verdict_header', lang));
|
|
1531
1436
|
for (const variant of targetVariants) {
|
|
1532
1437
|
const trace = saturation.perVariant[variant];
|
|
1533
1438
|
if (!trace || trace.length === 0) {
|
|
1534
|
-
console.log(
|
|
1439
|
+
console.log(tCli('cli.saturation.variant_no_trace', lang, { variant }));
|
|
1535
1440
|
continue;
|
|
1536
1441
|
}
|
|
1537
|
-
console.log(
|
|
1538
|
-
console.log(
|
|
1539
|
-
|
|
1442
|
+
console.log(tCli('cli.saturation.variant_label', lang, { variant }));
|
|
1443
|
+
console.log(tCli('cli.saturation.checkpoints', lang, {
|
|
1444
|
+
n: trace.length, list: trace.map((p) => p.n).join(', '),
|
|
1445
|
+
}));
|
|
1446
|
+
const last = trace[trace.length - 1];
|
|
1447
|
+
console.log(tCli('cli.saturation.last_point', lang, {
|
|
1448
|
+
mean: last.mean.toFixed(3), lo: last.ciLow.toFixed(3), hi: last.ciHigh.toFixed(3),
|
|
1449
|
+
}));
|
|
1540
1450
|
if (saturation.verdicts?.[variant]) {
|
|
1541
1451
|
const v = saturation.verdicts[variant];
|
|
1542
|
-
|
|
1452
|
+
const result = v.saturated
|
|
1453
|
+
? tCli('cli.saturation.persisted_verdict_saturated', lang, { n: v.atN ?? '?' })
|
|
1454
|
+
: tCli('cli.saturation.persisted_verdict_unsaturated', lang);
|
|
1455
|
+
console.log(tCli('cli.saturation.persisted_verdict', lang, {
|
|
1456
|
+
method: v.method, result, reason: v.reason,
|
|
1457
|
+
}));
|
|
1543
1458
|
}
|
|
1544
1459
|
else if (trace.length < 5) {
|
|
1545
|
-
console.log(
|
|
1460
|
+
console.log(tCli('cli.saturation.skipped_too_few_points', lang, { n: trace.length }));
|
|
1546
1461
|
}
|
|
1547
1462
|
}
|
|
1548
1463
|
console.log('');
|
|
@@ -1551,34 +1466,16 @@ async function handleSaturation(argv) {
|
|
|
1551
1466
|
// handleVerdict — one-line ship/no-ship verdict (v0.22)
|
|
1552
1467
|
// ---------------------------------------------------------------------------
|
|
1553
1468
|
async function handleVerdict(argv) {
|
|
1469
|
+
const lang = langFromArgv(argv);
|
|
1554
1470
|
const reportId = argv[0];
|
|
1555
1471
|
if (!reportId || reportId === '--help' || reportId === '-h') {
|
|
1556
|
-
console.log(
|
|
1557
|
-
'',
|
|
1558
|
-
'Usage: omk bench verdict <reportId> [options]',
|
|
1559
|
-
'',
|
|
1560
|
-
'聚合 bootstrap CI / 三层 ci-gate / saturation / human α 给出一行结论。',
|
|
1561
|
-
'',
|
|
1562
|
-
'Verdict 等级:',
|
|
1563
|
-
' PROGRESS 显著改进 + 三层全过',
|
|
1564
|
-
' CAUTIOUS 改进真实但有警告 (gate 破 / 幅度太小 / 控制组本身崩)',
|
|
1565
|
-
' REGRESS 显著回退 — 不要 ship',
|
|
1566
|
-
' NOISE CI 跨 0,无法判定',
|
|
1567
|
-
' UNDERPOWERED 样本不足,需要扩 N 重测',
|
|
1568
|
-
' SOLO 单变体报告,无对比对象',
|
|
1569
|
-
'',
|
|
1570
|
-
'Options:',
|
|
1571
|
-
' --reports-dir <dir> report store dir (default: ~/.oh-my-knowledge/reports)',
|
|
1572
|
-
' --threshold <num> 三层 gate 阈值 (default 3.5,匹配 omk bench ci)',
|
|
1573
|
-
' --trivial-diff <num> "幅度太小"阈值 (default 0.1)',
|
|
1574
|
-
' --verbose 展开 per-pair 详情',
|
|
1575
|
-
'',
|
|
1576
|
-
].join('\n'));
|
|
1472
|
+
console.log(tCli('cli.help.verdict', lang));
|
|
1577
1473
|
process.exit(reportId ? 0 : 1);
|
|
1578
1474
|
}
|
|
1579
1475
|
const { values } = parseArgs({
|
|
1580
1476
|
args: argv.slice(1),
|
|
1581
1477
|
options: {
|
|
1478
|
+
...COMMON_OPTIONS,
|
|
1582
1479
|
'reports-dir': { type: 'string', default: DEFAULT_REPORTS_DIR },
|
|
1583
1480
|
threshold: { type: 'string' },
|
|
1584
1481
|
'trivial-diff': { type: 'string' },
|
|
@@ -1590,12 +1487,12 @@ async function handleVerdict(argv) {
|
|
|
1590
1487
|
const store = createFileStore(resolve(values['reports-dir']));
|
|
1591
1488
|
const report = await store.get(reportId);
|
|
1592
1489
|
if (!report) {
|
|
1593
|
-
console.error(
|
|
1490
|
+
console.error(tCli('cli.common.report_not_found', lang, { id: reportId }));
|
|
1594
1491
|
process.exit(1);
|
|
1595
1492
|
}
|
|
1596
1493
|
const { computeVerdict, formatVerdictText } = await import('./eval-core/verdict.js');
|
|
1597
1494
|
const result = computeVerdict(report, {
|
|
1598
|
-
|
|
1495
|
+
gateThreshold: values.threshold != null ? Number(values.threshold) : undefined,
|
|
1599
1496
|
triviallySmallDiff: values['trivial-diff'] != null ? Number(values['trivial-diff']) : undefined,
|
|
1600
1497
|
});
|
|
1601
1498
|
console.log(formatVerdictText(result, { verbose: Boolean(values.verbose) }));
|
|
@@ -1614,31 +1511,16 @@ async function handleVerdict(argv) {
|
|
|
1614
1511
|
// handleDiagnose — per-sample quality diagnostics (v0.23 A)
|
|
1615
1512
|
// ---------------------------------------------------------------------------
|
|
1616
1513
|
async function handleDiagnose(argv) {
|
|
1514
|
+
const lang = langFromArgv(argv);
|
|
1617
1515
|
const reportId = argv[0];
|
|
1618
1516
|
if (!reportId || reportId === '--help' || reportId === '-h') {
|
|
1619
|
-
console.log(
|
|
1620
|
-
'',
|
|
1621
|
-
'Usage: omk bench diagnose <reportId> [options]',
|
|
1622
|
-
'',
|
|
1623
|
-
'诊断样本集本身的质量问题:区分度低 / 重复 / 歧义 / 成本异常 / 全 fail。',
|
|
1624
|
-
'回答"测评结论是否被坏样本污染"——与 omk bench verdict 互补。',
|
|
1625
|
-
'',
|
|
1626
|
-
'Options:',
|
|
1627
|
-
' --reports-dir <dir> report store dir',
|
|
1628
|
-
' --samples <path> 样本文件路径 (用于 near-duplicate 检测;默认从 report.meta.request 读)',
|
|
1629
|
-
' --top <n> 每类只显示前 N 个 (默认 10,0=全部)',
|
|
1630
|
-
' --duplicate-rouge <num> near-duplicate ROUGE-1 阈值 (默认 0.7)',
|
|
1631
|
-
' --ambiguous-stddev <num> 歧义阈值,judge stddev (默认 1.0,需要 --judge-repeat ≥ 2 数据)',
|
|
1632
|
-
' --cost-k <num> 成本异常倍数 vs median (默认 3)',
|
|
1633
|
-
' --latency-k <num> 耗时异常倍数 vs median (默认 3)',
|
|
1634
|
-
' --flat <num> flat_scores 分差阈值 (默认 0.5)',
|
|
1635
|
-
'',
|
|
1636
|
-
].join('\n'));
|
|
1517
|
+
console.log(tCli('cli.help.diagnose', lang));
|
|
1637
1518
|
process.exit(reportId ? 0 : 1);
|
|
1638
1519
|
}
|
|
1639
1520
|
const { values } = parseArgs({
|
|
1640
1521
|
args: argv.slice(1),
|
|
1641
1522
|
options: {
|
|
1523
|
+
...COMMON_OPTIONS,
|
|
1642
1524
|
'reports-dir': { type: 'string', default: DEFAULT_REPORTS_DIR },
|
|
1643
1525
|
samples: { type: 'string' },
|
|
1644
1526
|
top: { type: 'string', default: '10' },
|
|
@@ -1654,7 +1536,7 @@ async function handleDiagnose(argv) {
|
|
|
1654
1536
|
const store = createFileStore(resolve(values['reports-dir']));
|
|
1655
1537
|
const report = await store.get(reportId);
|
|
1656
1538
|
if (!report) {
|
|
1657
|
-
console.error(
|
|
1539
|
+
console.error(tCli('cli.common.report_not_found', lang, { id: reportId }));
|
|
1658
1540
|
process.exit(1);
|
|
1659
1541
|
}
|
|
1660
1542
|
// Try to read the samples file for near-duplicate detection. Source order:
|
|
@@ -1669,7 +1551,9 @@ async function handleDiagnose(argv) {
|
|
|
1669
1551
|
samples = loadSamples(samplesPath).samples;
|
|
1670
1552
|
}
|
|
1671
1553
|
catch (err) {
|
|
1672
|
-
process.stderr.write(
|
|
1554
|
+
process.stderr.write(tCli('cli.common.warn_load_samples_failed', lang, {
|
|
1555
|
+
path: samplesPath, message: err.message,
|
|
1556
|
+
}));
|
|
1673
1557
|
}
|
|
1674
1558
|
}
|
|
1675
1559
|
const topRaw = Number(values.top);
|
|
@@ -1694,29 +1578,16 @@ async function handleDiagnose(argv) {
|
|
|
1694
1578
|
// handleFailures — LLM-driven failure clustering (v0.23 B)
|
|
1695
1579
|
// ---------------------------------------------------------------------------
|
|
1696
1580
|
async function handleFailures(argv) {
|
|
1581
|
+
const lang = langFromArgv(argv);
|
|
1697
1582
|
const reportId = argv[0];
|
|
1698
1583
|
if (!reportId || reportId === '--help' || reportId === '-h') {
|
|
1699
|
-
console.log(
|
|
1700
|
-
'',
|
|
1701
|
-
'Usage: omk bench failures <reportId> [options]',
|
|
1702
|
-
'',
|
|
1703
|
-
'把已有 report 的失败样本喂给一次 LLM 调用,自动聚类 + 给修复建议。',
|
|
1704
|
-
'失败定义:compositeScore < threshold 或 ok=false。',
|
|
1705
|
-
'',
|
|
1706
|
-
'Options:',
|
|
1707
|
-
' --reports-dir <dir> report store dir',
|
|
1708
|
-
' --judge-executor <name> 执行器 (default: claude)',
|
|
1709
|
-
' --judge-model <id> 聚类用的 model (default: 沿用 report.meta.judgeModel)',
|
|
1710
|
-
' --max-clusters <n> 最多多少 cluster (default 5)',
|
|
1711
|
-
' --threshold <num> compositeScore < threshold 算失败 (default 3)',
|
|
1712
|
-
' --max-feed <n> 最多喂给 LLM 多少条 (default 50,超出取最差)',
|
|
1713
|
-
'',
|
|
1714
|
-
].join('\n'));
|
|
1584
|
+
console.log(tCli('cli.help.failures', lang));
|
|
1715
1585
|
process.exit(reportId ? 0 : 1);
|
|
1716
1586
|
}
|
|
1717
1587
|
const { values } = parseArgs({
|
|
1718
1588
|
args: argv.slice(1),
|
|
1719
1589
|
options: {
|
|
1590
|
+
...COMMON_OPTIONS,
|
|
1720
1591
|
'reports-dir': { type: 'string', default: DEFAULT_REPORTS_DIR },
|
|
1721
1592
|
'judge-executor': { type: 'string', default: 'claude' },
|
|
1722
1593
|
'judge-model': { type: 'string' },
|
|
@@ -1730,12 +1601,12 @@ async function handleFailures(argv) {
|
|
|
1730
1601
|
const store = createFileStore(resolve(values['reports-dir']));
|
|
1731
1602
|
const report = await store.get(reportId);
|
|
1732
1603
|
if (!report) {
|
|
1733
|
-
console.error(
|
|
1604
|
+
console.error(tCli('cli.common.report_not_found', lang, { id: reportId }));
|
|
1734
1605
|
process.exit(1);
|
|
1735
1606
|
}
|
|
1736
1607
|
const judgeModel = values['judge-model'] ?? report.meta?.judgeModel;
|
|
1737
1608
|
if (!judgeModel) {
|
|
1738
|
-
console.error('
|
|
1609
|
+
console.error(tCli('cli.common.no_judge_model', lang));
|
|
1739
1610
|
process.exit(1);
|
|
1740
1611
|
}
|
|
1741
1612
|
const { createExecutor } = await import('./executors/index.js');
|