oh-my-knowledge 0.27.0 → 0.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -34
- package/README.zh.md +31 -34
- package/dist/src/analysis/report-diagnostics.d.ts +2 -2
- package/dist/src/analysis/report-diagnostics.js +2 -2
- package/dist/src/analysis/sample-diagnostics.d.ts +3 -3
- package/dist/src/analysis/sample-diagnostics.js +3 -3
- package/dist/src/authoring/generator.js +1 -1
- package/dist/src/authoring/generator.js.map +1 -1
- package/dist/src/cli/commands/doctor.d.ts.map +1 -1
- package/dist/src/cli/commands/doctor.js +39 -2
- package/dist/src/cli/commands/doctor.js.map +1 -1
- package/dist/src/cli/commands/eval.d.ts.map +1 -1
- package/dist/src/cli/commands/eval.js +0 -5
- package/dist/src/cli/commands/eval.js.map +1 -1
- package/dist/src/cli/commands/{export.d.ts → evolve.d.ts} +1 -1
- package/dist/src/cli/commands/evolve.d.ts.map +1 -0
- package/dist/src/cli/commands/{improve-skill.js → evolve.js} +3 -3
- package/dist/src/cli/commands/evolve.js.map +1 -0
- package/dist/src/cli/commands/registry.d.ts.map +1 -1
- package/dist/src/cli/commands/registry.js +4 -22
- package/dist/src/cli/commands/registry.js.map +1 -1
- package/dist/src/cli/commands/{improve.d.ts → sample.d.ts} +1 -1
- package/dist/src/cli/commands/sample.d.ts.map +1 -0
- package/dist/src/cli/commands/{improve-samples.js → sample.js} +5 -4
- package/dist/src/cli/commands/sample.js.map +1 -0
- package/dist/src/cli/i18n-dict.d.ts +2 -4
- package/dist/src/cli/i18n-dict.d.ts.map +1 -1
- package/dist/src/cli/i18n-dict.js +165 -373
- package/dist/src/cli/i18n-dict.js.map +1 -1
- package/dist/src/cli/parse-run-config.d.ts +1 -1
- package/dist/src/cli/parse-run-config.js +1 -1
- package/dist/src/doctor/health/builtin-dimensions.d.ts +11 -0
- package/dist/src/doctor/health/builtin-dimensions.d.ts.map +1 -0
- package/dist/src/doctor/health/builtin-dimensions.js +94 -0
- package/dist/src/doctor/health/builtin-dimensions.js.map +1 -0
- package/dist/src/doctor/health/composer.d.ts +19 -0
- package/dist/src/doctor/health/composer.d.ts.map +1 -0
- package/dist/src/doctor/health/composer.js +289 -0
- package/dist/src/doctor/health/composer.js.map +1 -0
- package/dist/src/doctor/health/dimension-registry.d.ts +13 -0
- package/dist/src/doctor/health/dimension-registry.d.ts.map +1 -0
- package/dist/src/doctor/health/dimension-registry.js +28 -0
- package/dist/src/doctor/health/dimension-registry.js.map +1 -0
- package/dist/src/doctor/health/dimension-spec.d.ts +46 -0
- package/dist/src/doctor/health/dimension-spec.d.ts.map +1 -0
- package/dist/src/doctor/health/dimension-spec.js +12 -0
- package/dist/src/doctor/health/dimension-spec.js.map +1 -0
- package/dist/src/doctor/health/parser.d.ts +27 -0
- package/dist/src/doctor/health/parser.d.ts.map +1 -0
- package/dist/src/doctor/health/parser.js +190 -0
- package/dist/src/doctor/health/parser.js.map +1 -0
- package/dist/src/doctor/health/prompt-builder.d.ts +22 -0
- package/dist/src/doctor/health/prompt-builder.d.ts.map +1 -0
- package/dist/src/doctor/health/prompt-builder.js +162 -0
- package/dist/src/doctor/health/prompt-builder.js.map +1 -0
- package/dist/src/doctor/health/register.d.ts +13 -0
- package/dist/src/doctor/health/register.d.ts.map +1 -0
- package/dist/src/doctor/health/register.js +20 -0
- package/dist/src/doctor/health/register.js.map +1 -0
- package/dist/src/doctor/html-renderer.d.ts +20 -0
- package/dist/src/doctor/html-renderer.d.ts.map +1 -0
- package/dist/src/doctor/html-renderer.js +366 -0
- package/dist/src/doctor/html-renderer.js.map +1 -0
- package/dist/src/doctor/index.d.ts.map +1 -1
- package/dist/src/doctor/index.js +55 -6
- package/dist/src/doctor/index.js.map +1 -1
- package/dist/src/doctor/renderer.d.ts +4 -0
- package/dist/src/doctor/renderer.d.ts.map +1 -1
- package/dist/src/doctor/renderer.js +72 -18
- package/dist/src/doctor/renderer.js.map +1 -1
- package/dist/src/doctor/rules.d.ts +6 -5
- package/dist/src/doctor/rules.d.ts.map +1 -1
- package/dist/src/doctor/rules.js +4 -3
- package/dist/src/doctor/rules.js.map +1 -1
- package/dist/src/types/doctor.d.ts +41 -3
- package/dist/src/types/doctor.d.ts.map +1 -1
- package/dist/src/types/doctor.js +3 -0
- package/dist/src/types/doctor.js.map +1 -1
- package/dist/src/types/eval.d.ts +1 -1
- package/dist/src/types/report.d.ts +2 -2
- package/dist/src/types/report.d.ts.map +1 -1
- package/package.json +1 -1
- package/dist/src/cli/commands/eval-debias.d.ts +0 -2
- package/dist/src/cli/commands/eval-debias.d.ts.map +0 -1
- package/dist/src/cli/commands/eval-debias.js +0 -88
- package/dist/src/cli/commands/eval-debias.js.map +0 -1
- package/dist/src/cli/commands/export-diff.d.ts +0 -2
- package/dist/src/cli/commands/export-diff.d.ts.map +0 -1
- package/dist/src/cli/commands/export-diff.js +0 -177
- package/dist/src/cli/commands/export-diff.js.map +0 -1
- package/dist/src/cli/commands/export-saturation.d.ts +0 -2
- package/dist/src/cli/commands/export-saturation.d.ts.map +0 -1
- package/dist/src/cli/commands/export-saturation.js +0 -59
- package/dist/src/cli/commands/export-saturation.js.map +0 -1
- package/dist/src/cli/commands/export-verdict.d.ts +0 -2
- package/dist/src/cli/commands/export-verdict.d.ts.map +0 -1
- package/dist/src/cli/commands/export-verdict.js +0 -45
- package/dist/src/cli/commands/export-verdict.js.map +0 -1
- package/dist/src/cli/commands/export.d.ts.map +0 -1
- package/dist/src/cli/commands/export.js +0 -150
- package/dist/src/cli/commands/export.js.map +0 -1
- package/dist/src/cli/commands/improve-failures.d.ts +0 -2
- package/dist/src/cli/commands/improve-failures.d.ts.map +0 -1
- package/dist/src/cli/commands/improve-failures.js +0 -59
- package/dist/src/cli/commands/improve-failures.js.map +0 -1
- package/dist/src/cli/commands/improve-plan.d.ts +0 -2
- package/dist/src/cli/commands/improve-plan.d.ts.map +0 -1
- package/dist/src/cli/commands/improve-plan.js +0 -75
- package/dist/src/cli/commands/improve-plan.js.map +0 -1
- package/dist/src/cli/commands/improve-samples.d.ts +0 -2
- package/dist/src/cli/commands/improve-samples.d.ts.map +0 -1
- package/dist/src/cli/commands/improve-samples.js.map +0 -1
- package/dist/src/cli/commands/improve-skill.d.ts +0 -2
- package/dist/src/cli/commands/improve-skill.d.ts.map +0 -1
- package/dist/src/cli/commands/improve-skill.js.map +0 -1
- package/dist/src/cli/commands/improve.d.ts.map +0 -1
- package/dist/src/cli/commands/improve.js +0 -34
- package/dist/src/cli/commands/improve.js.map +0 -1
- package/dist/src/cli/coverage-renderer.d.ts +0 -15
- package/dist/src/cli/coverage-renderer.d.ts.map +0 -1
- package/dist/src/cli/coverage-renderer.js +0 -74
- package/dist/src/cli/coverage-renderer.js.map +0 -1
package/README.md
CHANGED
|
@@ -37,11 +37,11 @@ When the `omk` skill is available in Claude Code, you can invoke it directly lik
|
|
|
37
37
|
|
|
38
38
|
```bash
|
|
39
39
|
/omk eval # evaluate the artifact(s) in the current project
|
|
40
|
-
/omk
|
|
41
|
-
/omk
|
|
40
|
+
/omk evolve # auto-iterate to improve a skill
|
|
41
|
+
/omk sample # generate or fill test cases
|
|
42
42
|
```
|
|
43
43
|
|
|
44
|
-
You can also just say "compare v1 vs v2 for me" or "improve this artifact" and omk picks the right command.
|
|
44
|
+
These slash commands are natural-language entry points — the agent reads the conversation context to figure out which skill to operate on, so you usually don't pass the path explicitly. (See the Codex section below for the literal `omk evolve <skill>` CLI form.) You can also just say "compare v1 vs v2 for me" or "improve this artifact" and omk picks the right command.
|
|
45
45
|
|
|
46
46
|
### Use inside Codex
|
|
47
47
|
|
|
@@ -49,8 +49,8 @@ Codex does not support Claude Code style `/omk ...` slash commands by default. I
|
|
|
49
49
|
|
|
50
50
|
```bash
|
|
51
51
|
omk eval
|
|
52
|
-
omk
|
|
53
|
-
omk
|
|
52
|
+
omk evolve skills/my-skill.md
|
|
53
|
+
omk sample skills/my-skill.md
|
|
54
54
|
```
|
|
55
55
|
|
|
56
56
|
You can also describe the goal in natural language, such as "compare v1 vs v2" or "generate test cases for this skill".
|
|
@@ -61,7 +61,7 @@ Teams doing knowledge engineering produce lots of knowledge artifacts (skills to
|
|
|
61
61
|
|
|
62
62
|
## Key features
|
|
63
63
|
|
|
64
|
-
- **
|
|
64
|
+
- **LLM health audit** — `omk doctor` runs a single LLM session that emits a multi-dimension report; 7 builtin dimensions (trigger & boundary / doc clarity / instruction precision / dependency / tool conventions / security & compliance / example completeness) each get a *healthy / sub-healthy / unhealthy / N-A* grade plus findings and suggestions; dimensions are extensible, and `--html` produces a visual report. Pass `--static-only` for an offline mode (CI nodes without an LLM, debugging without network) that runs the static checks only (readability / metadata / dependencies / samples contract). `omk eval` still runs static readability / metadata / dependency gates internally to protect eval quality (separation of roles: doctor = audit, eval = evaluate)
|
|
65
65
|
- **Controlled-variable offline eval** — fix the model and samples, vary only the artifact; works with Claude Code skills, CLAUDE.md prompts, RAG knowledge bases, or any markdown-based instruction
|
|
66
66
|
- **Six-dimension scoring** — separate signals for Fact / Behavior / LLM-judge / Cost / Efficiency / Stability, so a regression in one axis isn't hidden by gains in another
|
|
67
67
|
- **Production session observability** — parse Claude Code session JSONL traces, measure per-skill failure rate, latency, token cost, and knowledge-gap signals on real user sessions
|
|
@@ -95,12 +95,10 @@ RAG-specific evals: see RAGAS (separate niche, complementary to omk). Full compa
|
|
|
95
95
|
| **Multi-executor** | Claude CLI / Claude SDK / Codex CLI / Codex SDK / OpenAI / Gemini / any custom command |
|
|
96
96
|
| **21+ assertion types** | substring, regex, JSON Schema, ROUGE/BLEU/Levenshtein similarity, agent tool-call assertions, semantic similarity, custom JS, and more |
|
|
97
97
|
| **Statistical rigor** | Bootstrap CI / Krippendorff α / length-debias / saturation curve — all on by default. [Details →](docs/statistical-rigor.md) |
|
|
98
|
-
| **Sample diagnostics** | `omk improve <id>` — 7 issue kinds (low discrimination / duplicates / ambiguous rubric / cost outliers / etc.) + 0-100 healthScore |
|
|
99
|
-
| **Failure clustering** | `omk improve failures <id>` — single LLM call clusters failed samples and emits per-cluster fixes |
|
|
100
98
|
| **RAG metrics** | `faithfulness` / `answer_relevancy` / `context_recall` — anti-hallucination + answer relevance + context coverage; auto-inherits length-debias |
|
|
101
99
|
| **Hard budget caps** | `--budget-usd / --budget-per-sample-usd / --budget-per-sample-ms` — abort on total-cost overrun, flag per-sample overruns; partial report persisted |
|
|
102
100
|
| **Construct-validity isolation** | `--strict-baseline` (default ON) cuts three contamination channels so baseline doesn't silently see the skill it's being compared against: (1) SDK skill auto-discovery, (2) subagent Skill tool, (3) cwd file-system access via the `skills/<name>/` symlink that's normally there for the treatment variant. eval.yaml `allowedSkills` for per-variant whitelists |
|
|
103
|
-
| **Sample design science** | Sample schema with `capability` / `difficulty` / `construct` / `provenance` metadata fields (HF Dataset Cards style)
|
|
101
|
+
| **Sample design science** | Sample schema with `capability` / `difficulty` / `construct` / `provenance` metadata fields (HF Dataset Cards style); studio surfaces coverage breakdown plus `rubric_clarity_low` / `capability_thin` flags. `omk sample` auto-stamps provenance on generated cases. See [docs/sample-design-spec.md](docs/sample-design-spec.md) for the 8 industry-gap mapping |
|
|
104
102
|
| **Multi-judge ensemble** | `--judge-models claude:opus,openai:gpt-4o` cross-vendor scoring + agreement metrics |
|
|
105
103
|
| **MCP URL fetching** | pull content from private-doc URLs via an MCP server (SSO-protected knowledge bases, etc.) |
|
|
106
104
|
| **Blind A/B** | `--blind` hides variant names; HTML report has a reveal button |
|
|
@@ -359,7 +357,7 @@ Reports display results across six independent dimensions. The three scoring lay
|
|
|
359
357
|
|
|
360
358
|
## CLI reference
|
|
361
359
|
|
|
362
|
-
omk exposes a workflow CLI for knowledge artifacts.
|
|
360
|
+
omk exposes a workflow CLI for knowledge artifacts. Seven top-level commands cover the full loop: `init` (scaffold) · `doctor` (static check) · `eval` (offline A/B) · `observe` (online trace) · `evolve` (auto-iterate a skill) · `sample` (generate or fill test cases) · `studio` (local web UI for reports & analysis).
|
|
363
361
|
|
|
364
362
|
### `omk init`
|
|
365
363
|
|
|
@@ -372,22 +370,28 @@ Scaffolds an evaluation project with two starter skill variants and an `eval-sam
|
|
|
372
370
|
### `omk doctor`
|
|
373
371
|
|
|
374
372
|
```bash
|
|
375
|
-
omk doctor
|
|
376
|
-
omk doctor skills/v1.md
|
|
377
|
-
omk doctor skills/ --
|
|
378
|
-
omk doctor --
|
|
373
|
+
omk doctor # audit current dir / ./skills
|
|
374
|
+
omk doctor skills/v1.md # audit one skill file
|
|
375
|
+
omk doctor skills/ --html report.html # produce a visual HTML report
|
|
376
|
+
omk doctor skills/ --json > r.json # JSON for CI / external tools
|
|
377
|
+
omk doctor --gate; echo $? # silent gate; exit 1 on fatal failures, warnings do not block
|
|
378
|
+
omk doctor --static-only # offline mode: static checks only, no LLM call
|
|
379
379
|
```
|
|
380
380
|
|
|
381
|
-
|
|
381
|
+
LLM health audit: a single LLM session emits per-dimension grades, findings, and suggestions for the 7 builtin dimensions; the HTML report sorts dimensions fail→warn→pass→skipped with errors first within each dim. Dimensions are extensible — call `registerHealthDimension` in your own code and the new section is folded into the same LLM call's prompt and report (order = registration order).
|
|
382
|
+
|
|
383
|
+
Static-only mode (`--static-only`): for CI nodes without claude / codex installed, or local debugging without network — runs the four static rules (readability / metadata / dependencies / samples contract) with zero LLM calls and zero cost. Output goes through the same `DoctorReport` shape and combines with `--json` / `--gate` / `--html`.
|
|
384
|
+
|
|
385
|
+
`omk eval` still runs its own static readability / metadata / dependency / samples-contract gates internally to protect eval quality; that path is separate from this user-facing `omk doctor` command and the two roles do not overlap.
|
|
382
386
|
|
|
383
387
|
### `omk eval`
|
|
384
388
|
|
|
385
389
|
```bash
|
|
386
|
-
omk eval --control
|
|
390
|
+
omk eval --control baseline --treatment my-skill # single-skill necessity test (baseline = reserved "no skill" variant)
|
|
391
|
+
omk eval --control code-review-v1 --treatment code-review-v2 # multi-variant A/B
|
|
387
392
|
omk eval --config eval.yaml
|
|
388
393
|
omk eval --batch
|
|
389
394
|
omk eval gold compare <report-id> --gold-dir gold-dataset
|
|
390
|
-
omk eval debias length <report-id>
|
|
391
395
|
```
|
|
392
396
|
|
|
393
397
|
Runs the offline evaluation, applies the verdict gate, persists the report, and returns a ship/no-ship exit code. Bootstrap CI is enabled by default on this workflow.
|
|
@@ -415,7 +419,7 @@ Common options:
|
|
|
415
419
|
--trivial-diff <num> practically tiny diff cutoff (default 0.1)
|
|
416
420
|
--report-only produce the report and print verdict, but always exit 0
|
|
417
421
|
--no-gate alias for --report-only
|
|
418
|
-
--skip-connectivity skip model connectivity check
|
|
422
|
+
--skip-connectivity skip model connectivity check (internal static gates still run)
|
|
419
423
|
--no-serve do not auto-start the report server after evaluation
|
|
420
424
|
```
|
|
421
425
|
|
|
@@ -431,30 +435,23 @@ omk observe ~/.claude/projects/my-project --kb /path/to/project
|
|
|
431
435
|
|
|
432
436
|
Turns real Claude Code session traces into skill-health reports: knowledge usage, gap signals, execution stability, tokens, and latency. This is production observation, not production scoring.
|
|
433
437
|
|
|
434
|
-
### `omk
|
|
438
|
+
### `omk evolve`
|
|
435
439
|
|
|
436
440
|
```bash
|
|
437
|
-
omk
|
|
438
|
-
omk
|
|
439
|
-
omk improve failures <report-id> # cluster failed samples into root causes
|
|
440
|
-
omk improve samples [skill] # generate or fill eval samples
|
|
441
|
-
omk improve skill <skill> # iterate a skill through eval-driven rewrites
|
|
441
|
+
omk evolve <skill> # multi-round auto-iteration on a skill
|
|
442
|
+
omk evolve skills/foo.md --rounds 10 --target 4.5
|
|
442
443
|
```
|
|
443
444
|
|
|
444
|
-
|
|
445
|
+
Auto-iterates a skill through repeated eval → judge → rewrite loops until it hits `--target` or exhausts `--rounds`. Cost scales with `rounds × samples × variants`; a typical run takes minutes to tens of minutes. Original skill files are versioned under `skills/evolve/*.r0.md`.
|
|
445
446
|
|
|
446
|
-
### `omk
|
|
447
|
+
### `omk sample`
|
|
447
448
|
|
|
448
449
|
```bash
|
|
449
|
-
omk
|
|
450
|
-
omk
|
|
451
|
-
omk export <report-id> --format github-summary
|
|
452
|
-
omk export diff <report-id> --regressions-only
|
|
453
|
-
omk export verdict <report-id>
|
|
454
|
-
omk export saturation <report-id>
|
|
450
|
+
omk sample <skill> # generate or fill eval-samples test cases for one skill
|
|
451
|
+
omk sample --batch # generate for skills missing eval-samples
|
|
455
452
|
```
|
|
456
453
|
|
|
457
|
-
|
|
454
|
+
One-shot generation. Auto-stamps `provenance` on generated cases. Generated assertions use English, numbers, or code tokens so they compare cleanly across bilingual outputs.
|
|
458
455
|
|
|
459
456
|
### `omk studio`
|
|
460
457
|
|
|
@@ -465,7 +462,7 @@ omk studio --reports-dir ~/.oh-my-knowledge/reports
|
|
|
465
462
|
omk studio --no-open
|
|
466
463
|
```
|
|
467
464
|
|
|
468
|
-
Starts the local knowledge workbench for browsing reports and observation analyses.
|
|
465
|
+
Starts the local knowledge workbench for browsing reports and observation analyses. Verdict, sample diffs, regressions, saturation curves, and per-sample drill-downs all live in the studio UI — there is no CLI export / analysis subcommand. For CI gates, use `omk eval`'s exit code (0 on `PROGRESS`, non-zero otherwise) or `jq` over the report JSON.
|
|
469
466
|
|
|
470
467
|
## Executors
|
|
471
468
|
|
package/README.zh.md
CHANGED
|
@@ -37,11 +37,11 @@ omk eval --control code-review-v1 --treatment code-review-v2 # → 5 分钟
|
|
|
37
37
|
|
|
38
38
|
```bash
|
|
39
39
|
/omk eval # 评测当前项目的 artifact
|
|
40
|
-
/omk
|
|
41
|
-
/omk
|
|
40
|
+
/omk evolve # 多轮自动迭代改进 skill
|
|
41
|
+
/omk sample # 生成或补齐评测用例
|
|
42
42
|
```
|
|
43
43
|
|
|
44
|
-
|
|
44
|
+
这些 slash command 是自然语言入口 —— agent 会从对话上下文里推断要操作哪个 skill,所以一般不用显式传路径。(下面 Codex 段落给出 `omk evolve <skill>` 的裸 CLI 形式。)也可以直接说「帮我评测 v1 和 v2 的差异」、「改进一下这个 artifact」,omk 会自动理解意图并调用对应命令。
|
|
45
45
|
|
|
46
46
|
### 在 Codex 中使用
|
|
47
47
|
|
|
@@ -49,8 +49,8 @@ Codex 默认不支持 `/omk ...` 这种 Claude Code 风格的 slash command。
|
|
|
49
49
|
|
|
50
50
|
```bash
|
|
51
51
|
omk eval
|
|
52
|
-
omk
|
|
53
|
-
omk
|
|
52
|
+
omk evolve skills/my-skill.md
|
|
53
|
+
omk sample skills/my-skill.md
|
|
54
54
|
```
|
|
55
55
|
|
|
56
56
|
也可以直接用自然语言描述目标,例如"比较 v1 和 v2 的评测差异"、"为这个 skill 生成评测用例"。
|
|
@@ -61,7 +61,7 @@ omk improve samples skills/my-skill.md
|
|
|
61
61
|
|
|
62
62
|
## 核心能力
|
|
63
63
|
|
|
64
|
-
-
|
|
64
|
+
- **LLM 健康度审计** — `omk doctor` 用单次 LLM 会话产出多维度健康度报告,7 个内置维度(触发与边界 / 文档清晰 / 指令精确性 / 依赖检查 / 工具规范 / 安全与合规 / 示例完备)独立给「健康 / 亚健康 / 不健康 / 不适用」+ findings + 改进建议;维度可扩展,`--html` 产可视化报告。无 LLM 环境(CI 节点 / 断网调试)可加 `--static-only` 跑纯静态检查(可读性 / 元数据 / 依赖 / samples 契约)。`omk eval` 内部仍跑静态可读性、元数据、依赖 gate 把关评测质量(角色分离:doctor=审计,eval=评测)
|
|
65
65
|
- **控制变量离线评测** — 固定模型和用例,只变知识载体;兼容 Claude Code skill、CLAUDE.md prompt、RAG 知识库等任何 markdown 形式的指令
|
|
66
66
|
- **六维独立打分** — Fact / Behavior / LLM-judge / Cost / Efficiency / Stability 分别出信号,单一维度的回退不会被其他维度的收益掩盖
|
|
67
67
|
- **线上 session 观测** — 解析 Claude Code session JSONL,在真实用户会话上测量各 skill 的失败率、耗时、token 成本和知识缺口信号
|
|
@@ -95,12 +95,10 @@ RAG 专项评测请看 RAGAS(独立 niche,跟 omk 互补)。完整对比
|
|
|
95
95
|
| **多执行器** | 支持 Claude CLI / Claude SDK / Codex CLI / Codex SDK / OpenAI / Gemini 及自定义命令 |
|
|
96
96
|
| **21+ 种断言** | 包含子串、正则、JSON Schema、ROUGE/BLEU/Levenshtein 相似度、Agent 工具调用、语义相似度、自定义函数等 |
|
|
97
97
|
| **统计严谨性** | Bootstrap CI / Krippendorff α / 长度去偏 / 饱和曲线 —— 全部默认开。[详情 →](docs/zh/statistical-rigor.md) |
|
|
98
|
-
| **用例质量诊断** | `omk improve <id>` 7 类 issue(区分度低 / 重复 / 歧义 / 成本异常 / 全 fail 等)+ healthScore 0-100 |
|
|
99
|
-
| **失败聚类 + 根因** | `omk improve failures <id>` 单 LLM 调用聚类失败用例 + 每 cluster 给修复建议 |
|
|
100
98
|
| **RAG metrics** | `faithfulness` / `answer_relevancy` / `context_recall` 三 metric — 反幻觉 + 切题度 + context 覆盖,自动继承长度去偏 |
|
|
101
99
|
| **预算硬阈值** | `--budget-usd / --budget-per-sample-usd / --budget-per-sample-ms` 总成本 + 单用例成本/耗时上限,超出中止保留 partial report |
|
|
102
100
|
| **用例隔离 (construct validity)** | `--strict-baseline` (默认开) 三堵 baseline 拿到被测 skill 的污染路径:(1) SDK skill auto-discovery (2) subagent Skill 工具调用 (3) cwd 文件系统(避免 baseline 顺 `skills/<name>/` symlink 直接 Read 到 SKILL.md)。eval.yaml `allowedSkills` 支持 per-variant 白名单 |
|
|
103
|
-
| **用例设计科学性 (sample design science)** | Sample schema 加 `capability` / `difficulty` / `construct` / `provenance` 元数据字段(HF Dataset Cards
|
|
101
|
+
| **用例设计科学性 (sample design science)** | Sample schema 加 `capability` / `difficulty` / `construct` / `provenance` 元数据字段(HF Dataset Cards 风),studio 输出 coverage 分桶 + `rubric_clarity_low` / `capability_thin` 两类 issue。`omk sample` 自动给生成的用例打 provenance。详见 [docs/sample-design-spec.md](docs/sample-design-spec.md),含 8 条行业 gap(HELM / MMLU-Pro / Construct Validity / IRT / Dataset Cards / Adversarial)的 omk v1 映射 |
|
|
104
102
|
| **多评委 ensemble** | `--judge-models claude:opus,openai:gpt-4o` 跨厂商评分 + agreement 度量 |
|
|
105
103
|
| **MCP URL 获取** | 通过 MCP Server 获取私有文档 URL 内容(SSO 保护的知识库等) |
|
|
106
104
|
| **盲测 A/B** | `--blind` 隐藏变体名称,HTML 报告有揭晓按钮 |
|
|
@@ -359,7 +357,7 @@ export default function(output, { sample, assertion }) {
|
|
|
359
357
|
|
|
360
358
|
## CLI 参考
|
|
361
359
|
|
|
362
|
-
omk 的公开 CLI
|
|
360
|
+
omk 的公开 CLI 由 7 个顶层命令构成完整闭环:`init`(脚手架)·`doctor`(静态检查)·`eval`(离线 A/B 评测)·`observe`(线上 trace 观测)·`evolve`(多轮自动迭代 skill)·`sample`(生成或补齐评测用例)·`studio`(本地 Web 工作台,看报告 / 分析)。
|
|
363
361
|
|
|
364
362
|
### `omk init`
|
|
365
363
|
|
|
@@ -372,22 +370,28 @@ omk init [目录]
|
|
|
372
370
|
### `omk doctor`
|
|
373
371
|
|
|
374
372
|
```bash
|
|
375
|
-
omk doctor
|
|
376
|
-
omk doctor skills/v1.md
|
|
377
|
-
omk doctor skills/ --
|
|
378
|
-
omk doctor --
|
|
373
|
+
omk doctor # 体检当前目录或 ./skills
|
|
374
|
+
omk doctor skills/v1.md # 体检单个 skill
|
|
375
|
+
omk doctor skills/ --html report.html # 产 HTML 可视化报告
|
|
376
|
+
omk doctor skills/ --json > r.json # JSON 给 CI / 外部工具消费
|
|
377
|
+
omk doctor --gate; echo $? # 静默门禁,fatal 问题 exit 1,警告不阻断
|
|
378
|
+
omk doctor --static-only # 离线模式:只跑静态检查,不调 LLM
|
|
379
379
|
```
|
|
380
380
|
|
|
381
|
-
|
|
381
|
+
LLM 健康度审计:单次 LLM 会话产出 7 个内置维度的健康度评分 + findings + 改进建议;HTML 报告按 fail→warn→pass→skipped 排序,错误 finding 优先。维度可扩展(在自己代码里调 `registerHealthDimension`,自动并入同一次 LLM 调用的 prompt 与报告,顺序 = 注册顺序)。
|
|
382
|
+
|
|
383
|
+
离线静态模式(`--static-only`):CI 节点没装 claude / codex、本地断网调试等场景下跑 4 条静态 rule(可读性 / 元数据 / 依赖 / samples 契约),零 LLM 调用、零成本。结果同样进 `DoctorReport`,可与 `--json` / `--gate` / `--html` 组合。
|
|
384
|
+
|
|
385
|
+
`omk eval` 内部继续跑静态 readability / metadata / dependency / samples 契约 gate 保护评测质量,这条路径与用户入口的 `omk doctor` 角色分离,互不干扰。
|
|
382
386
|
|
|
383
387
|
### `omk eval`
|
|
384
388
|
|
|
385
389
|
```bash
|
|
386
|
-
omk eval --control
|
|
390
|
+
omk eval --control baseline --treatment my-skill # 单 skill 必要性测试(baseline 是保留 variant,代表「不注入 skill」)
|
|
391
|
+
omk eval --control code-review-v1 --treatment code-review-v2 # 多版本 A/B
|
|
387
392
|
omk eval --config eval.yaml
|
|
388
393
|
omk eval --batch
|
|
389
394
|
omk eval gold compare <report-id> --gold-dir gold-dataset
|
|
390
|
-
omk eval debias length <report-id>
|
|
391
395
|
```
|
|
392
396
|
|
|
393
397
|
运行离线评测,应用 verdict gate,持久化报告,并用 exit code 表示 ship/no-ship。这个工作流默认开启 bootstrap CI。
|
|
@@ -415,7 +419,7 @@ omk eval debias length <report-id>
|
|
|
415
419
|
--trivial-diff <num> 实际可忽略 diff 阈值(默认 0.1)
|
|
416
420
|
--report-only 生成报告并打印 verdict,但始终 exit 0
|
|
417
421
|
--no-gate --report-only 的别名
|
|
418
|
-
--skip-connectivity
|
|
422
|
+
--skip-connectivity 跳过模型连通性检查(内部静态 gate 仍强制执行)
|
|
419
423
|
--no-serve 评测后不自动启动报告服务
|
|
420
424
|
```
|
|
421
425
|
|
|
@@ -431,30 +435,23 @@ omk observe ~/.claude/projects/my-project --kb /path/to/project
|
|
|
431
435
|
|
|
432
436
|
把真实 Claude Code session trace 转成 skill 健康度报告:知识使用、gap 信号、执行稳定性、token 和耗时。这是生产观测,不是生产评分。
|
|
433
437
|
|
|
434
|
-
### `omk
|
|
438
|
+
### `omk evolve`
|
|
435
439
|
|
|
436
440
|
```bash
|
|
437
|
-
omk
|
|
438
|
-
omk
|
|
439
|
-
omk improve failures <report-id> # 聚类失败样本并给根因
|
|
440
|
-
omk improve samples [skill] # 生成或补齐 eval samples
|
|
441
|
-
omk improve skill <skill> # 通过评测循环迭代 skill
|
|
441
|
+
omk evolve <skill> # 多轮自动迭代 skill
|
|
442
|
+
omk evolve skills/foo.md --rounds 10 --target 4.5
|
|
442
443
|
```
|
|
443
444
|
|
|
444
|
-
|
|
445
|
+
让 skill 跑 eval → judge → 改写 SKILL.md 的多轮闭环,直到达到 `--target` 或 `--rounds` 上限。耗时按 `轮数 × 样本 × 变体` 累加,几分钟到几十分钟级别。原始 skill 文件版本保存在 `skills/evolve/*.r0.md`。
|
|
445
446
|
|
|
446
|
-
### `omk
|
|
447
|
+
### `omk sample`
|
|
447
448
|
|
|
448
449
|
```bash
|
|
449
|
-
omk
|
|
450
|
-
omk
|
|
451
|
-
omk export <report-id> --format github-summary
|
|
452
|
-
omk export diff <report-id> --regressions-only
|
|
453
|
-
omk export verdict <report-id>
|
|
454
|
-
omk export saturation <report-id>
|
|
450
|
+
omk sample <skill> # 为单个 skill 生成或补齐评测用例
|
|
451
|
+
omk sample --batch # 为目录下缺评测集的 skill 批量生成
|
|
455
452
|
```
|
|
456
453
|
|
|
457
|
-
|
|
454
|
+
一次性生成。自动给生成的用例打 `provenance`。生成的 assertions 使用英文 / 数字 / 代码 token,便于跨中英文输出对比。
|
|
458
455
|
|
|
459
456
|
### `omk studio`
|
|
460
457
|
|
|
@@ -465,7 +462,7 @@ omk studio --reports-dir ~/.oh-my-knowledge/reports
|
|
|
465
462
|
omk studio --no-open
|
|
466
463
|
```
|
|
467
464
|
|
|
468
|
-
|
|
465
|
+
启动本地知识工作台浏览报告。verdict、样本回退、跨样本 diff、饱和曲线、单样本 drill-down 全部在 studio UI 里 —— omk 不提供 CLI 导出 / 分析子命令。CI gate 用 `omk eval` 的 exit code(PROGRESS 退 0、其他非 0),需要文字摘要自己 `jq` report JSON。
|
|
469
466
|
|
|
470
467
|
## 执行器
|
|
471
468
|
|
|
@@ -26,8 +26,8 @@ export declare function analyzeResults(report: Report, opts?: AnalyzeResultsOpti
|
|
|
26
26
|
* Missing fields are bucketed under the `unspecified` key in the relevant
|
|
27
27
|
* distribution map, so users see "I have N samples without difficulty declared".
|
|
28
28
|
*
|
|
29
|
-
*
|
|
30
|
-
* in grading / judge / verdict. See docs/sample-design-spec.md.
|
|
29
|
+
* Persisted on the report so studio can surface coverage gaps. Does NOT
|
|
30
|
+
* participate in grading / judge / verdict. See docs/sample-design-spec.md.
|
|
31
31
|
*/
|
|
32
32
|
export declare function buildSampleQualityAggregate(samples: Sample[]): SampleQualityAggregate;
|
|
33
33
|
export declare function generateAnalysisSummary(report: Report, lang?: Lang): string | undefined;
|
|
@@ -56,8 +56,8 @@ export function analyzeResults(report, opts = {}) {
|
|
|
56
56
|
* Missing fields are bucketed under the `unspecified` key in the relevant
|
|
57
57
|
* distribution map, so users see "I have N samples without difficulty declared".
|
|
58
58
|
*
|
|
59
|
-
*
|
|
60
|
-
* in grading / judge / verdict. See docs/sample-design-spec.md.
|
|
59
|
+
* Persisted on the report so studio can surface coverage gaps. Does NOT
|
|
60
|
+
* participate in grading / judge / verdict. See docs/sample-design-spec.md.
|
|
61
61
|
*/
|
|
62
62
|
export function buildSampleQualityAggregate(samples) {
|
|
63
63
|
const capabilityCoverage = {};
|
|
@@ -16,8 +16,8 @@
|
|
|
16
16
|
* (d) are cost / latency outliers (one sample spikes the budget by itself), or
|
|
17
17
|
* (e) fail across the board (likely broken sample, not a model deficiency).
|
|
18
18
|
*
|
|
19
|
-
*
|
|
20
|
-
* the SAMPLE SET
|
|
19
|
+
* Persisted on the report so studio can surface these to users iterating
|
|
20
|
+
* the SAMPLE SET (alongside iterating the SKILL).
|
|
21
21
|
*
|
|
22
22
|
* Output shape: a flat list of issues, each tagged with severity and the
|
|
23
23
|
* minimal evidence needed to act ("here are the matching prompts"). Issues are
|
|
@@ -75,7 +75,7 @@ export declare function normalizeCapability(raw: string): string;
|
|
|
75
75
|
type DiagnosticLang = 'zh' | 'en';
|
|
76
76
|
export declare function formatSampleIssue(issue: SampleIssue, lang?: DiagnosticLang): string;
|
|
77
77
|
/**
|
|
78
|
-
* Plain-text formatter for
|
|
78
|
+
* Plain-text formatter for sample-quality diagnostics output.
|
|
79
79
|
*/
|
|
80
80
|
export declare function formatSampleDiagnostics(diag: SampleDiagnosticReport, options?: {
|
|
81
81
|
topN?: number;
|
|
@@ -16,8 +16,8 @@
|
|
|
16
16
|
* (d) are cost / latency outliers (one sample spikes the budget by itself), or
|
|
17
17
|
* (e) fail across the board (likely broken sample, not a model deficiency).
|
|
18
18
|
*
|
|
19
|
-
*
|
|
20
|
-
* the SAMPLE SET
|
|
19
|
+
* Persisted on the report so studio can surface these to users iterating
|
|
20
|
+
* the SAMPLE SET (alongside iterating the SKILL).
|
|
21
21
|
*
|
|
22
22
|
* Output shape: a flat list of issues, each tagged with severity and the
|
|
23
23
|
* minimal evidence needed to act ("here are the matching prompts"). Issues are
|
|
@@ -376,7 +376,7 @@ export function formatSampleIssue(issue, lang = 'zh') {
|
|
|
376
376
|
}
|
|
377
377
|
}
|
|
378
378
|
/**
|
|
379
|
-
* Plain-text formatter for
|
|
379
|
+
* Plain-text formatter for sample-quality diagnostics output.
|
|
380
380
|
*/
|
|
381
381
|
export function formatSampleDiagnostics(diag, options = {}) {
|
|
382
382
|
const lines = [];
|
|
@@ -58,7 +58,7 @@ ${skillContent}
|
|
|
58
58
|
// (see sanitizeGeneratedSamples).
|
|
59
59
|
const { stripped } = sanitizeGeneratedSamples(samples);
|
|
60
60
|
if (stripped.length > 0) {
|
|
61
|
-
process.stderr.write(`[omk
|
|
61
|
+
process.stderr.write(`[omk sample] LLM-output 含 ${stripped.length} 个非法元数据字段,已剥离避免污染:\n - ${stripped.join('\n - ')}\n`);
|
|
62
62
|
}
|
|
63
63
|
return { samples, costUSD: result.costUSD };
|
|
64
64
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"generator.js","sourceRoot":"","sources":["../../../src/authoring/generator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAGtE,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;uCA2BiB,CAAC;AASxC,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,EAAE,YAAY,EAAE,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,aAAa,EAAE,YAAY,GAAG,QAAQ,EAA0B;IACvI,MAAM,QAAQ,GAAG,cAAc,CAAC,YAAY,CAAC,CAAC;IAE9C,MAAM,MAAM,GAAG;;EAEf,YAAY;;iBAEG,KAAK,sBAAsB,CAAC;IAE3C,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC,CAAC;IAExE,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,sBAAsB,MAAM,CAAC,KAAK,IAAI,eAAe,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED,kEAAkE;IAClE,IAAI,OAAO,GAAG,MAAM,CAAC,MAAO,CAAC,IAAI,EAAE,CAAC;IACpC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAChE,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAChC,CAAC;IAED,IAAI,OAAiB,CAAC;IACtB,IAAI,CAAC;QACH,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;IACvE,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,CAAC;IAED,8EAA8E;IAC9E,kCAAkC;IAClC,MAAM,EAAE,QAAQ,EAAE,GAAG,wBAAwB,CAAC,OAAO,CAAC,CAAC;IACvD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,
|
|
1
|
+
{"version":3,"file":"generator.js","sourceRoot":"","sources":["../../../src/authoring/generator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAGtE,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;uCA2BiB,CAAC;AASxC,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,EAAE,YAAY,EAAE,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,aAAa,EAAE,YAAY,GAAG,QAAQ,EAA0B;IACvI,MAAM,QAAQ,GAAG,cAAc,CAAC,YAAY,CAAC,CAAC;IAE9C,MAAM,MAAM,GAAG;;EAEf,YAAY;;iBAEG,KAAK,sBAAsB,CAAC;IAE3C,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC,CAAC;IAExE,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,sBAAsB,MAAM,CAAC,KAAK,IAAI,eAAe,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED,kEAAkE;IAClE,IAAI,OAAO,GAAG,MAAM,CAAC,MAAO,CAAC,IAAI,EAAE,CAAC;IACpC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAChE,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAChC,CAAC;IAED,IAAI,OAAiB,CAAC;IACtB,IAAI,CAAC;QACH,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;IACvE,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,CAAC;IAED,8EAA8E;IAC9E,kCAAkC;IAClC,MAAM,EAAE,QAAQ,EAAE,GAAG,wBAAwB,CAAC,OAAO,CAAC,CAAC;IACvD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,6BAA6B,QAAQ,CAAC,MAAM,2BAA2B,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CACnG,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;AAC9C,CAAC;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,UAAU,wBAAwB,CAAC,OAAiB;IACxD,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;IAC7D,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,eAAe,EAAE,kBAAkB,CAAC,CAAC,CAAC;IACjF,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;QACvC,kEAAkE;QAClE,0DAA0D;QAC1D,IAAI,OAAO,CAAC,CAAC,SAAS,KAAK,QAAQ,IAAI,CAAC,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChE,CAAC,CAAC,SAAS,GAAG,IAAI,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;QACrD,CAAC;QACD,IAAI,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1D,MAAM,IAAI,KAAK,CAAC,WAAW,CAAC,mDAAmD,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;QACrG,CAAC;QAED,IAAI,CAAC,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YAC/B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;gBACtG,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,iBAAiB,OAAO,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC;gBACnE,OAAQ,CAA8B,CAAC,UAAU,CAAC;YACpD,CAAC;QACH,CAAC;QACD,IAAI,CAAC,CAAC,UAAU,KAAK,SAAS,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,UAAoB,CAAC,EAAE,CAAC;YAChF,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,gBAAgB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAC1E,OAAQ,CAA8B,CAAC,UAAU,CAAC;QACpD,CAAC;QACD,IAAI,CAAC,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,CAAC,SAAS,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC;YACnF,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,gBAAgB,OAAO,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC;YACjE,OAAQ,CAA6B,CAAC,SAAS,CAAC;QAClD,CAAC;QACD,IAAI,CAAC,CAAC,UAAU,KAAK,SAAS,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,UAAoB,CAAC,EAAE,CAAC;YAChF,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,gBAAgB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;YAC1E,OAAQ,CAA8B,CAAC,UAAU,CAAC;QACpD,CAAC;QACD,kFAAkF;QAClF,IAAI,CAAC,CAAC,CAAC,UAAU;YAAE,CAAC,CAAC,UAAU,GAAG,eAAe,CAAC;IACpD,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,CAAC;AACtB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"doctor.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/doctor.ts"],"names":[],"mappings":"AAoDA,wBAAsB,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"doctor.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/doctor.ts"],"names":[],"mappings":"AAoDA,wBAAsB,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAgI3D"}
|
|
@@ -60,14 +60,24 @@ export async function execute(argv) {
|
|
|
60
60
|
model: { type: 'string' },
|
|
61
61
|
samples: { type: 'string' },
|
|
62
62
|
timeout: { type: 'string' },
|
|
63
|
+
html: { type: 'string' },
|
|
64
|
+
'static-only': { type: 'boolean', default: false },
|
|
63
65
|
},
|
|
64
66
|
});
|
|
65
67
|
const target = positionals[0] ?? null;
|
|
66
68
|
const executorName = values.executor ?? 'claude';
|
|
67
69
|
const model = values.model ?? 'sonnet';
|
|
68
70
|
const timeoutRaw = values.timeout;
|
|
69
|
-
|
|
70
|
-
|
|
71
|
+
// 默认 LLM 健康度审计(7 内置维度 + 用户注册的自定义维度);--static-only 切到
|
|
72
|
+
// 离线静态模式:只跑 4 条静态 rule(skill_readable / skill_metadata /
|
|
73
|
+
// dependencies_present / samples_contract_aligned),不调 LLM。后者用于
|
|
74
|
+
// CI 节点没装 claude/codex、本地断网调试等场景。
|
|
75
|
+
const staticOnly = values['static-only'];
|
|
76
|
+
const runHealthCheck = !staticOnly;
|
|
77
|
+
// 单次 LLM 会话(7+N 维度,内部多 turn)默认 timeout 600s(10 min)。
|
|
78
|
+
const defaultTimeoutSec = 600;
|
|
79
|
+
const timeoutSec = timeoutRaw != null ? Number(timeoutRaw) : defaultTimeoutSec;
|
|
80
|
+
const timeoutMs = Math.max(1000, Math.floor((Number.isFinite(timeoutSec) ? timeoutSec : defaultTimeoutSec) * 1000));
|
|
71
81
|
const cwd = process.cwd();
|
|
72
82
|
const samplesPath = values.samples ? resolve(values.samples) : findDefaultSamplesPath(target, cwd);
|
|
73
83
|
let samples;
|
|
@@ -84,8 +94,20 @@ export async function execute(argv) {
|
|
|
84
94
|
process.stderr.write(tCli('cli.common.warn_load_samples_failed', lang, { path: samplesPath, message: msg }));
|
|
85
95
|
}
|
|
86
96
|
}
|
|
97
|
+
// 副作用 import: 注册 7 内置维度 spec + skill_health composer rule。
|
|
98
|
+
// 用户在自己代码 / plugin 里 import dimension-registry 注册自定义维度,
|
|
99
|
+
// 同样会被 composer 拼到 prompt(顺序 = 注册顺序)。
|
|
100
|
+
await import('../../doctor/health/register.js');
|
|
87
101
|
const { runDoctor } = await import('../../doctor/index.js');
|
|
88
102
|
const { renderDoctorReportText, renderDoctorReportJson } = await import('../../doctor/renderer.js');
|
|
103
|
+
// 默认模式过滤掉静态 rule 只留 composer;--static-only 反过来:只留静态 rule
|
|
104
|
+
// 跳过 composer。静态 rule 在 omk eval 里继续当强制 gate,doctor 这条线只
|
|
105
|
+
// 选择性暴露给用户。
|
|
106
|
+
const { getRegisteredRules } = await import('../../doctor/rules.js');
|
|
107
|
+
const { isComposerRule } = await import('../../types/doctor.js');
|
|
108
|
+
const rulesOverride = staticOnly
|
|
109
|
+
? getRegisteredRules().filter((r) => !isComposerRule(r))
|
|
110
|
+
: getRegisteredRules().filter(isComposerRule);
|
|
89
111
|
let report;
|
|
90
112
|
try {
|
|
91
113
|
report = await runDoctor({
|
|
@@ -95,6 +117,8 @@ export async function execute(argv) {
|
|
|
95
117
|
model,
|
|
96
118
|
timeoutMs,
|
|
97
119
|
lang,
|
|
120
|
+
runHealthCheck,
|
|
121
|
+
rules: rulesOverride,
|
|
98
122
|
samples,
|
|
99
123
|
requires,
|
|
100
124
|
});
|
|
@@ -114,6 +138,19 @@ export async function execute(argv) {
|
|
|
114
138
|
}
|
|
115
139
|
const isJson = values.json;
|
|
116
140
|
const isGate = values.gate;
|
|
141
|
+
const htmlPath = values.html;
|
|
142
|
+
// --html 与 --json/--gate 可以共存:--html 写文件;同时 stdout/stderr 仍按
|
|
143
|
+
// 主输出格式跑(--json → JSON 到 stdout / --gate → 静默 / 默认 → text 到 stderr)。
|
|
144
|
+
if (htmlPath) {
|
|
145
|
+
const { renderDoctorReportHtml } = await import('../../doctor/html-renderer.js');
|
|
146
|
+
const { writeFileSync, mkdirSync } = await import('node:fs');
|
|
147
|
+
const { dirname, resolve } = await import('node:path');
|
|
148
|
+
const abs = resolve(htmlPath);
|
|
149
|
+
mkdirSync(dirname(abs), { recursive: true });
|
|
150
|
+
writeFileSync(abs, renderDoctorReportHtml(report, lang), 'utf8');
|
|
151
|
+
// 通知用户(stderr,不污染 --json 的 stdout)
|
|
152
|
+
console.error(lang === 'zh' ? `HTML 报告已写入: ${abs}` : `HTML report written to: ${abs}`);
|
|
153
|
+
}
|
|
117
154
|
if (isJson) {
|
|
118
155
|
console.log(renderDoctorReportJson(report));
|
|
119
156
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"doctor.js","sourceRoot":"","sources":["../../../../src/cli/commands/doctor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAI3D,MAAM,wBAAwB,GAAG,CAAC,mBAAmB,EAAE,mBAAmB,EAAE,kBAAkB,CAAU,CAAC;AAEzG,SAAS,gBAAgB,CAAC,GAAW;IACnC,KAAK,MAAM,IAAI,IAAI,wBAAwB,EAAE,CAAC;QAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAClC,IAAI,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO,SAAS,CAAC;IAC9C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAqB,EAAE,GAAW;IAC1D,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,GAAG,GAAG,CAAC,GAAW,EAAQ,EAAE;QAChC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1C,CAAC,CAAC;IACF,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QAClC,IAAI,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC;YACjC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;gBACvB,GAAG,CAAC,SAAS,CAAC,CAAC;gBACf,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;gBACxB,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;YACnC,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;gBAClC,GAAG,CAAC,MAAM,CAAC,CAAC;gBACZ,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;IACH,CAAC;IACD,GAAG,CAAC,GAAG,CAAC,CAAC;IACT,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,sBAAsB,CAAC,MAAqB,EAAE,GAAW;IAChE,KAAK,MAAM,GAAG,IAAI,gBAAgB,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;QAChD,MAAM,WAAW,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAC1C,IAAI,WAAW;YAAE,OAAO,WAAW,CAAC;IACtC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,IAAc;IAC1C,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IAChC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,qBAAqB,CAAC;QACpD,IAAI,EAAE,IAAI;QACV,gBAAgB,EAAE,IAAI;QACtB,OAAO,EAAE;YACP,GAAG,cAAc;YACjB,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;YACzC,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;YACzC,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC5B,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACzB,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC3B,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;
|
|
1
|
+
{"version":3,"file":"doctor.js","sourceRoot":"","sources":["../../../../src/cli/commands/doctor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAI3D,MAAM,wBAAwB,GAAG,CAAC,mBAAmB,EAAE,mBAAmB,EAAE,kBAAkB,CAAU,CAAC;AAEzG,SAAS,gBAAgB,CAAC,GAAW;IACnC,KAAK,MAAM,IAAI,IAAI,wBAAwB,EAAE,CAAC;QAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAClC,IAAI,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO,SAAS,CAAC;IAC9C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAqB,EAAE,GAAW;IAC1D,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,GAAG,GAAG,CAAC,GAAW,EAAQ,EAAE;QAChC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1C,CAAC,CAAC;IACF,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QAClC,IAAI,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC;YACjC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;gBACvB,GAAG,CAAC,SAAS,CAAC,CAAC;gBACf,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;gBACxB,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;YACnC,CAAC;iBAAM,CAAC;gBACN,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;gBAClC,GAAG,CAAC,MAAM,CAAC,CAAC;gBACZ,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;IACH,CAAC;IACD,GAAG,CAAC,GAAG,CAAC,CAAC;IACT,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,sBAAsB,CAAC,MAAqB,EAAE,GAAW;IAChE,KAAK,MAAM,GAAG,IAAI,gBAAgB,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;QAChD,MAAM,WAAW,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAC1C,IAAI,WAAW;YAAE,OAAO,WAAW,CAAC;IACtC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,IAAc;IAC1C,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IAChC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,qBAAqB,CAAC;QACpD,IAAI,EAAE,IAAI;QACV,gBAAgB,EAAE,IAAI;QACtB,OAAO,EAAE;YACP,GAAG,cAAc;YACjB,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;YACzC,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;YACzC,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC5B,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACzB,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC3B,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC3B,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YACxB,aAAa,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;SACnD;KACF,CAAC,CAAC;IAEH,MAAM,MAAM,GAAkB,WAAW,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;IACrD,MAAM,YAAY,GAAI,MAAM,CAAC,QAA+B,IAAI,QAAQ,CAAC;IACzE,MAAM,KAAK,GAAI,MAAM,CAAC,KAA4B,IAAI,QAAQ,CAAC;IAC/D,MAAM,UAAU,GAAG,MAAM,CAAC,OAA6B,CAAC;IACxD,qDAAqD;IACrD,yDAAyD;IACzD,+DAA+D;IAC/D,kCAAkC;IAClC,MAAM,UAAU,GAAG,MAAM,CAAC,aAAa,CAAY,CAAC;IACpD,MAAM,cAAc,GAAG,CAAC,UAAU,CAAC;IACnC,qDAAqD;IACrD,MAAM,iBAAiB,GAAG,GAAG,CAAC;IAC9B,MAAM,UAAU,GAAG,UAAU,IAAI,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC;IAC/E,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,iBAAiB,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;IACpH,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAC1B,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAiB,CAAC,CAAC,CAAC,CAAC,sBAAsB,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC7G,IAAI,OAA6B,CAAC;IAClC,IAAI,QAA4C,CAAC;IACjD,IAAI,WAAW,EAAE,CAAC;QAChB,IAAI,CAAC;YACH,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,8BAA8B,CAAC,CAAC;YACrE,MAAM,MAAM,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC;YACxC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;YACzB,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QAC7B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,qCAAqC,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;QAC/G,CAAC;IACH,CAAC;IAED,2DAA2D;IAC3D,wDAAwD;IACxD,sCAAsC;IACtC,MAAM,MAAM,CAAC,iCAAiC,CAAC,CAAC;IAEhD,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,uBAAuB,CAAC,CAAC;IAC5D,MAAM,EAAE,sBAAsB,EAAE,sBAAsB,EAAE,GAAG,MAAM,MAAM,CAAC,0BAA0B,CAAC,CAAC;IAEpG,yDAAyD;IACzD,yDAAyD;IACzD,YAAY;IACZ,MAAM,EAAE,kBAAkB,EAAE,GAAG,MAAM,MAAM,CAAC,uBAAuB,CAAC,CAAC;IACrE,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,uBAAuB,CAAC,CAAC;IACjE,MAAM,aAAa,GAAG,UAAU;QAC9B,CAAC,CAAC,kBAAkB,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;QACxD,CAAC,CAAC,kBAAkB,EAAE,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;IAEhD,IAAI,MAAM,CAAC;IACX,IAAI,CAAC;QACH,MAAM,GAAG,MAAM,SAAS,CAAC;YACvB,MAAM;YACN,GAAG;YACH,YAAY;YACZ,KAAK;YACL,SAAS;YACT,IAAI;YACJ,cAAc;YACd,KAAK,EAAE,aAAa;YACpB,OAAO;YACP,QAAQ;SACT,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,0DAA0D;QAC1D,IAAI,GAAG,YAAY,OAAO;YAAE,MAAM,GAAG,CAAC;QACtC,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC7D,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,2BAA2B,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC;QAChF,OAAO,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC;QAC1B,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,2BAA2B,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC;QAChF,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,IAAe,CAAC;IACtC,MAAM,MAAM,GAAG,MAAM,CAAC,IAAe,CAAC;IACtC,MAAM,QAAQ,GAAG,MAAM,CAAC,IAA0B,CAAC;IAEnD,6DAA6D;IAC7D,qEAAqE;IACrE,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,EAAE,sBAAsB,EAAE,GAAG,MAAM,MAAM,CAAC,+BAA+B,CAAC,CAAC;QACjF,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;QAC7D,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;QACvD,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC9B,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7C,aAAa,CAAC,GAAG,EAAE,sBAAsB,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;QACjE,mCAAmC;QACnC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,eAAe,GAAG,EAAE,CAAC,CAAC,CAAC,2BAA2B,GAAG,EAAE,CAAC,CAAC;IACzF,CAAC;IAED,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC,CAAC;IAC9C,CAAC;SAAM,IAAI,MAAM,EAAE,CAAC;QAClB,2DAA2D;QAC3D,IAAI,MAAM,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YAChC,MAAM,OAAO,GAAG,IAAI,KAAK,IAAI;gBAC3B,CAAC,CAAC,kBAAkB,MAAM,CAAC,MAAM,CAAC,IAAI,iBAAiB,MAAM,CAAC,MAAM,CAAC,IAAI,WAAW,MAAM,CAAC,MAAM,CAAC,IAAI,QAAQ;gBAC9G,CAAC,CAAC,kBAAkB,MAAM,CAAC,MAAM,CAAC,IAAI,yBAAyB,MAAM,CAAC,MAAM,CAAC,IAAI,WAAW,MAAM,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC;YACzH,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;SAAM,CAAC;QACN,IAAI,WAAW,IAAI,OAAO,EAAE,CAAC;YAC3B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,6BAA6B,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;QAChG,CAAC;QACD,sBAAsB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACvC,CAAC;IAED,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACzD,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/eval.ts"],"names":[],"mappings":"AAGA,wBAAsB,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAQ3D"}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { execute as debias } from './eval-debias.js';
|
|
2
1
|
import { execute as gold } from './eval-gold.js';
|
|
3
2
|
import { execute as run } from './eval-runner.js';
|
|
4
3
|
export async function execute(argv) {
|
|
@@ -7,10 +6,6 @@ export async function execute(argv) {
|
|
|
7
6
|
await gold(rest);
|
|
8
7
|
return;
|
|
9
8
|
}
|
|
10
|
-
if (sub === 'debias') {
|
|
11
|
-
await debias(rest);
|
|
12
|
-
return;
|
|
13
|
-
}
|
|
14
9
|
await run(argv);
|
|
15
10
|
}
|
|
16
11
|
//# sourceMappingURL=eval.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval.js","sourceRoot":"","sources":["../../../../src/cli/commands/eval.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,IAAI,
|
|
1
|
+
{"version":3,"file":"eval.js","sourceRoot":"","sources":["../../../../src/cli/commands/eval.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,IAAI,IAAI,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,OAAO,IAAI,GAAG,EAAE,MAAM,kBAAkB,CAAC;AAElD,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,IAAc;IAC1C,MAAM,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;IAC5B,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;QACnB,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;QACjB,OAAO;IACT,CAAC;IAED,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC;AAClB,CAAC"}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
export declare function execute(argv: string[]): Promise<void>;
|
|
2
|
-
//# sourceMappingURL=
|
|
2
|
+
//# sourceMappingURL=evolve.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"evolve.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/evolve.ts"],"names":[],"mappings":"AA0CA,wBAAsB,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CA4G3D"}
|
|
@@ -25,7 +25,7 @@ export async function execute(argv) {
|
|
|
25
25
|
allowPositionals: true,
|
|
26
26
|
});
|
|
27
27
|
// skill path 走 parseArgs 的 positionals,避免 raw argv.find 把 flag value
|
|
28
|
-
// 当成 path 误识别,例如 `
|
|
28
|
+
// 当成 path 误识别,例如 `omk evolve --judge-models openai-api:gpt-4o foo.md`。
|
|
29
29
|
const skillPath = positionals[0];
|
|
30
30
|
if (!skillPath) {
|
|
31
31
|
console.error(tCli('cli.evolve.specify_skill_path', lang));
|
|
@@ -42,7 +42,7 @@ export async function execute(argv) {
|
|
|
42
42
|
const { parseJudgeModelsArgOrExit } = await import('../parse-run-config.js');
|
|
43
43
|
const evolveJudges = parseJudgeModelsArgOrExit(values['judge-models']);
|
|
44
44
|
if (evolveJudges.length > 1) {
|
|
45
|
-
console.error(tCli('cli.common.judge_models_single_only', lang, { cmd: '
|
|
45
|
+
console.error(tCli('cli.common.judge_models_single_only', lang, { cmd: 'omk evolve' }));
|
|
46
46
|
throw new CliExit(2);
|
|
47
47
|
}
|
|
48
48
|
process.stderr.write(tCli('cli.evolve.section_header', lang, { path: skillPath }));
|
|
@@ -112,4 +112,4 @@ export async function execute(argv) {
|
|
|
112
112
|
throw new CliExit(1);
|
|
113
113
|
}
|
|
114
114
|
}
|
|
115
|
-
//# sourceMappingURL=
|
|
115
|
+
//# sourceMappingURL=evolve.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"evolve.js","sourceRoot":"","sources":["../../../../src/cli/commands/evolve.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAoChD,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,IAAc;IAC1C,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IAChC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,qBAAqB,CAAC;QACpD,IAAI,EAAE,IAAI;QACV,OAAO,EAAE;YACP,GAAG,cAAc;YACjB,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE;YACxC,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;YAC1B,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,mBAAmB,EAAE;YACzD,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE;YAC5C,cAAc,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,cAAc,EAAE;YAC3D,eAAe,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE;YACtD,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE;YAC7C,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE;YAC3C,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE;YAC/C,mBAAmB,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE;SACzD;QACD,gBAAgB,EAAE,IAAI;KACvB,CAAC,CAAC;IAEH,qEAAqE;IACrE,uEAAuE;IACvE,MAAM,SAAS,GAAuB,WAAW,CAAC,CAAC,CAAC,CAAC;IACrD,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,+BAA+B,EAAE,IAAI,CAAC,CAAC,CAAC;QAC3D,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IAED,IAAI,WAAW,GAAY,MAAM,CAAC,OAAkB,IAAI,mBAAmB,CAAC;IAC5E,IAAI,WAAW,KAAK,mBAAmB,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC;QAC7E,IAAI,UAAU,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;YAAE,WAAW,GAAG,mBAAmB,CAAC;aAC3E,IAAI,UAAU,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;YAAE,WAAW,GAAG,kBAAkB,CAAC;IACrF,CAAC;IAED,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,4BAA4B,CAAC,CAAC;IACnE,MAAM,EAAE,yBAAyB,EAAE,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC,CAAC;IAE7E,MAAM,YAAY,GAAG,yBAAyB,CAAC,MAAM,CAAC,cAAc,CAAW,CAAC,CAAC;IACjF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,qCAAqC,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC;QACxF,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,2BAA2B,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;IAEnF,IAAI,CAAC;QACH,MAAM,MAAM,GAAiB,MAAM,WAAW,CAAC;YAC7C,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC;YAC7B,WAAW,EAAE,OAAO,CAAC,WAAW,CAAC;YACjC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC/C,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI;YACpD,KAAK,EAAE,MAAM,CAAC,KAAe;YAC7B,WAAW,EAAE,YAAY;YACzB,YAAY,EAAE,MAAM,CAAC,eAAe,CAAW;YAC/C,YAAY,EAAE,MAAM,CAAC,QAAkB;YACvC,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YACzD,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,GAAG,IAAI;YAC5D,gBAAgB,EAAE,MAAM,CAAC,mBAAmB,CAAY;YACxD,UAAU,EAAE,cAAc,CAAC,IAAI,CAAgC;YAC/D,eAAe,CAAC,EAAE,KAAK,EAAE,WAAW,EAAE,YAAY,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE,KAAK,EAAqB;gBAClI,kEAAkE;gBAClE,6BAA6B;gBAC7B,MAAM,YAAY,GAAG,CAAC,CAAS,EAAE,CAAU,EAAU,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;gBACrF,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;oBACzB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,2BAA2B,EAAE,IAAI,EAAE;wBAC3D,KAAK,EAAE,KAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,YAAY,CAAC,OAAQ,EAAE,YAAY,KAAK,KAAK,CAAC;qBAC/E,CAAC,CAAC,CAAC;gBACN,CAAC;qBAAM,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;oBAC7B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,wBAAwB,EAAE,IAAI,EAAE;wBACxD,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;qBAClC,CAAC,CAAC,CAAC;gBACN,CAAC;qBAAM,IAAI,KAAK,KAAK,MAAM,EAAE,CAAC;oBAC5B,MAAM,MAAM,GAAW,KAAM,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,KAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;oBACjF,MAAM,MAAM,GAAW,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC;oBAC1D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,uBAAuB,EAAE,IAAI,EAAE;wBACvD,KAAK,EAAE,KAAK,EAAE,KAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,CAAC,OAAQ,EAAE,YAAY,KAAK,KAAK,CAAC;qBAC7G,CAAC,CAAC,CAAC;gBACN,CAAC;YACH,CAAC;SACF,CAAC,CAAC;QAEH,MAAM,WAAW,GAAW,MAAM,CAAC,UAAU,GAAG,CAAC;YAC/C,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,MAAM,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YAChF,CAAC,CAAC,GAAG,CAAC;QACR,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,KAAK,KAAK;YAChD,CAAC,CAAC,GAAG,CAAE,qDAAqD;YAC5D,CAAC,CAAC,IAAI,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QACzC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,oBAAoB,EAAE,IAAI,EAAE;YACpD,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC;YACxE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,CAAC,WAAW,EAAE,IAAI,EAAE,YAAY;SACrE,CAAC,CAAC,CAAC;QACJ,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,sBAAsB,EAAE,IAAI,EAAE;YACtD,IAAI,EAAE,MAAM,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,SAAS,CAAC;SACvD,CAAC,CAAC,CAAC;QACJ,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,2BAA2B,EAAE,IAAI,EAAE;YAC3D,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,QAAQ,CAAC;SAC9C,CAAC,CAAC,CAAC;QACJ,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACpB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,wBAAwB,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QACtF,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/C,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,iDAAiD;QACjD,IAAI,GAAG,YAAY,OAAO;YAAE,MAAM,GAAG,CAAC;QACtC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,yBAAyB,EAAE,IAAI,EAAE,EAAE,OAAO,EAAG,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAC1F,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;AACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/registry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AASrD,MAAM,WAAW,aAAa;IAC5B,+DAA+D;IAC/D,OAAO,EAAE,aAAa,CAAC;IACvB,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IACxC,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5C;AAED,iCAAiC;AACjC,eAAO,MAAM,gBAAgB,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,
|
|
1
|
+
{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/registry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AASrD,MAAM,WAAW,aAAa;IAC5B,+DAA+D;IAC/D,OAAO,EAAE,aAAa,CAAC;IACvB,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IACxC,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5C;AAED,iCAAiC;AACjC,eAAO,MAAM,gBAAgB,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAc1D,CAAC"}
|