oh-my-knowledge 1.0.0-beta.6 → 1.0.0-beta.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (247) hide show
  1. package/dist/assets/agent-skills/omk/references/commands.md +1 -2
  2. package/dist/cli/commands/eval/gold/index.js +2 -8
  3. package/dist/cli/commands/eval/index.d.ts +11 -3
  4. package/dist/cli/commands/eval/index.js +31 -19
  5. package/dist/cli/commands/evolve.js +5 -13
  6. package/dist/cli/commands/init.js +1 -2
  7. package/dist/cli/commands/observe/inbox.js +3 -1
  8. package/dist/cli/commands/promote.js +1 -13
  9. package/dist/cli/commands/rollback.js +1 -13
  10. package/dist/cli/lib/actor.d.ts +2 -0
  11. package/dist/cli/lib/actor.js +13 -0
  12. package/dist/cli/lib/cmd-flags.d.ts +1 -2
  13. package/dist/cli/lib/cmd-flags.js +1 -1
  14. package/dist/cli/lib/i18n-dict/common.d.ts +1 -1
  15. package/dist/cli/lib/i18n-dict/common.js +0 -12
  16. package/dist/cli/lib/i18n-dict/help.d.ts +1 -1
  17. package/dist/cli/lib/i18n-dict/help.js +0 -226
  18. package/dist/cli/lib/i18n-dict/run.d.ts +1 -1
  19. package/dist/cli/lib/i18n-dict/run.js +0 -156
  20. package/dist/cli/lib/i18n.d.ts +0 -10
  21. package/dist/cli/lib/i18n.js +0 -12
  22. package/dist/cli/lib/parse-run-config/judge-models.d.ts +0 -21
  23. package/dist/cli/lib/parse-run-config/judge-models.js +1 -43
  24. package/dist/cli/lib/parse-run-config/samples-discovery.d.ts +1 -1
  25. package/dist/cli/lib/parse-run-config/samples-discovery.js +3 -3
  26. package/dist/cli/lib/prepare-evaluation.d.ts +15 -0
  27. package/dist/cli/lib/prepare-evaluation.js +45 -0
  28. package/dist/cli/lib/progress.d.ts +1 -24
  29. package/dist/cli/lib/progress.js +1 -61
  30. package/dist/cli/lib/run-core-evaluation.d.ts +2 -9
  31. package/dist/cli/lib/run-core-evaluation.js +14 -44
  32. package/dist/cli/oclif/i18n.js +4 -7
  33. package/dist/diagnosis/observe-mapper.d.ts +1 -2
  34. package/dist/diagnosis/observe-mapper.js +0 -9
  35. package/dist/dsh-plugin/core-command.js +2 -55
  36. package/dist/eval-core/analysis/decision.d.ts +1 -3
  37. package/dist/eval-core/analysis/decision.js +0 -6
  38. package/dist/eval-core/analysis/runtime.d.ts +1 -3
  39. package/dist/eval-core/analysis/runtime.js +0 -6
  40. package/dist/eval-core/contracts/comparability.d.ts +0 -1
  41. package/dist/eval-core/contracts/comparability.js +0 -9
  42. package/dist/eval-core/contracts/json.d.ts +0 -1
  43. package/dist/eval-core/contracts/json.js +0 -9
  44. package/dist/eval-core/evaluation/runtime.d.ts +1 -3
  45. package/dist/eval-core/evaluation/runtime.js +0 -6
  46. package/dist/eval-core/execution/runtime.d.ts +0 -3
  47. package/dist/eval-core/execution/runtime.js +4 -10
  48. package/dist/eval-workflows/analysis/bootstrap.d.ts +0 -7
  49. package/dist/eval-workflows/analysis/bootstrap.js +0 -10
  50. package/dist/eval-workflows/assertions/deterministic.d.ts +0 -4
  51. package/dist/eval-workflows/assertions/deterministic.js +0 -19
  52. package/dist/eval-workflows/gold/human.d.ts +0 -28
  53. package/dist/eval-workflows/gold/human.js +1 -29
  54. package/dist/eval-workflows/hosts/adapters/claude/cli-protocol.js +4 -52
  55. package/dist/eval-workflows/hosts/adapters/claude/core-protocol-contract.d.ts +5 -0
  56. package/dist/eval-workflows/hosts/adapters/claude/core-protocol-contract.js +57 -0
  57. package/dist/eval-workflows/hosts/adapters/claude/resources.js +1 -16
  58. package/dist/eval-workflows/hosts/adapters/claude/sdk-protocol.js +5 -53
  59. package/dist/eval-workflows/hosts/adapters/codex/cli-protocol.d.ts +0 -2
  60. package/dist/eval-workflows/hosts/adapters/codex/cli-protocol.js +1 -3
  61. package/dist/eval-workflows/hosts/adapters/codex/cli.d.ts +1 -1
  62. package/dist/eval-workflows/hosts/adapters/codex/cli.js +1 -1
  63. package/dist/eval-workflows/hosts/adapters/codex/sdk-protocol.d.ts +0 -2
  64. package/dist/eval-workflows/hosts/adapters/codex/sdk-protocol.js +1 -3
  65. package/dist/eval-workflows/hosts/adapters/codex/sdk.d.ts +1 -1
  66. package/dist/eval-workflows/hosts/adapters/codex/sdk.js +1 -1
  67. package/dist/eval-workflows/hosts/adapters/shared/readonly-map-snapshot.d.ts +2 -0
  68. package/dist/eval-workflows/hosts/adapters/shared/readonly-map-snapshot.js +17 -0
  69. package/dist/eval-workflows/hosts/application.d.ts +2 -1
  70. package/dist/eval-workflows/hosts/application.js +12 -7
  71. package/dist/eval-workflows/hosts/composition/assembly.js +1 -16
  72. package/dist/eval-workflows/hosts/composition/node-runtime.js +2 -55
  73. package/dist/eval-workflows/hosts/composition/runtime-registry.js +1 -16
  74. package/dist/eval-workflows/hosts/composition/runtime.d.ts +2 -9
  75. package/dist/eval-workflows/hosts/composition/runtime.js +3 -49
  76. package/dist/eval-workflows/hosts/evaluators/executor-judge-invocation.d.ts +5 -0
  77. package/dist/eval-workflows/hosts/evaluators/executor-judge-invocation.js +57 -0
  78. package/dist/eval-workflows/hosts/input-resolution/node-cli-evaluation-resolver.js +63 -28
  79. package/dist/eval-workflows/hosts/resource-leases/node.d.ts +0 -5
  80. package/dist/eval-workflows/hosts/resource-leases/node.js +1 -20
  81. package/dist/eval-workflows/input-compilation/error.d.ts +1 -1
  82. package/dist/eval-workflows/input-compilation/parse.d.ts +2 -0
  83. package/dist/eval-workflows/input-compilation/parse.js +79 -94
  84. package/dist/eval-workflows/input-compilation/registry.d.ts +1 -1
  85. package/dist/eval-workflows/input-compilation/registry.js +24 -40
  86. package/dist/eval-workflows/inputs/assertion-types.d.ts +0 -1
  87. package/dist/eval-workflows/inputs/assertion-types.js +0 -1
  88. package/dist/eval-workflows/inputs/contracts/config-schema.d.ts +375 -0
  89. package/dist/eval-workflows/inputs/contracts/config-schema.js +173 -0
  90. package/dist/eval-workflows/inputs/contracts/config.d.ts +1 -82
  91. package/dist/eval-workflows/inputs/eval-config.d.ts +1 -1
  92. package/dist/eval-workflows/inputs/eval-config.js +22 -384
  93. package/dist/eval-workflows/inputs/judge-models.d.ts +2 -0
  94. package/dist/eval-workflows/inputs/judge-models.js +23 -0
  95. package/dist/eval-workflows/inputs/load-samples.d.ts +2 -8
  96. package/dist/eval-workflows/inputs/load-samples.js +12 -30
  97. package/dist/eval-workflows/inputs/sample-document.d.ts +0 -6
  98. package/dist/eval-workflows/inputs/sample-document.js +2 -70
  99. package/dist/eval-workflows/inputs/sample-locator.d.ts +0 -1
  100. package/dist/eval-workflows/inputs/sample-locator.js +0 -51
  101. package/dist/eval-workflows/measurement/analysis/agreement-source-adapter-v1.js +2 -69
  102. package/dist/eval-workflows/measurement/analysis/agreement-source-adapter.js +2 -69
  103. package/dist/eval-workflows/measurement/analysis/agreement-source-pairs.d.ts +9 -0
  104. package/dist/eval-workflows/measurement/analysis/agreement-source-pairs.js +72 -0
  105. package/dist/eval-workflows/measurement/analysis/bootstrap-family-table.d.ts +0 -1
  106. package/dist/eval-workflows/measurement/analysis/bootstrap-family-table.js +0 -3
  107. package/dist/eval-workflows/measurement/analysis/dimension-binding.d.ts +7 -0
  108. package/dist/eval-workflows/measurement/analysis/dimension-binding.js +17 -0
  109. package/dist/eval-workflows/measurement/analysis/dimension-table-v1.d.ts +0 -4
  110. package/dist/eval-workflows/measurement/analysis/dimension-table-v1.js +3 -28
  111. package/dist/eval-workflows/measurement/analysis/dimension-table.js +1 -16
  112. package/dist/eval-workflows/orchestration/evaluation-service.js +1 -0
  113. package/dist/eval-workflows/orchestration/orchestration.d.ts +0 -40
  114. package/dist/eval-workflows/orchestration/orchestration.js +0 -84
  115. package/dist/eval-workflows/orchestration/workflow.d.ts +1 -1
  116. package/dist/evidence/storage/default-dirs.d.ts +0 -2
  117. package/dist/evidence/storage/default-dirs.js +0 -2
  118. package/dist/evidence/storage/discovery-index.d.ts +0 -2
  119. package/dist/evidence/storage/discovery-index.js +0 -4
  120. package/dist/evidence/storage/file-names.d.ts +0 -2
  121. package/dist/evidence/storage/file-names.js +0 -6
  122. package/dist/executors/contracts/tool-call-status-schema.d.ts +7 -0
  123. package/dist/executors/contracts/tool-call-status-schema.js +7 -0
  124. package/dist/executors/contracts/trace-source-schema.d.ts +9 -0
  125. package/dist/executors/contracts/trace-source-schema.js +9 -0
  126. package/dist/executors/contracts/trace-source.d.ts +3 -1
  127. package/dist/executors/contracts/trace.d.ts +3 -1
  128. package/dist/executors/core/capabilities.d.ts +0 -5
  129. package/dist/executors/core/capabilities.js +3 -26
  130. package/dist/executors/core/registry.d.ts +11 -4
  131. package/dist/executors/core/registry.js +0 -9
  132. package/dist/executors/core/trace-source-kind.js +2 -10
  133. package/dist/executors/index.d.ts +1 -1
  134. package/dist/executors/index.js +1 -1
  135. package/dist/executors/mock-runtime/runtime.d.ts +0 -3
  136. package/dist/executors/mock-runtime/runtime.js +1 -9
  137. package/dist/executors/result-validation.d.ts +0 -1
  138. package/dist/executors/result-validation.js +0 -5
  139. package/dist/executors/tool-call-status.d.ts +0 -1
  140. package/dist/executors/tool-call-status.js +2 -10
  141. package/dist/observability/analysis/coverage-analyzer.d.ts +0 -1
  142. package/dist/observability/analysis/coverage-analyzer.js +0 -48
  143. package/dist/observability/analysis/gap-analyzer.d.ts +0 -8
  144. package/dist/observability/analysis/gap-analyzer.js +0 -15
  145. package/dist/observability/contracts/experience-enums.d.ts +227 -0
  146. package/dist/observability/contracts/experience-enums.js +227 -0
  147. package/dist/observability/contracts/experience-evidence-schema.d.ts +33130 -0
  148. package/dist/observability/contracts/experience-evidence-schema.js +704 -0
  149. package/dist/observability/contracts/experience.d.ts +75 -646
  150. package/dist/observability/contracts/inbox.d.ts +4 -4
  151. package/dist/observability/contracts/problem-patterns.d.ts +6 -30
  152. package/dist/observability/contracts/trace-metadata-schema.d.ts +10 -0
  153. package/dist/observability/contracts/trace-metadata-schema.js +10 -0
  154. package/dist/observability/contracts/trace.d.ts +3 -12
  155. package/dist/observability/conversation/catalog.d.ts +0 -1
  156. package/dist/observability/conversation/catalog.js +1 -4
  157. package/dist/observability/experience/report-codec.d.ts +8 -22
  158. package/dist/observability/experience/report-codec.js +22 -33
  159. package/dist/observability/experience/report-reference-validator.d.ts +2 -2
  160. package/dist/observability/experience/report-reference-validator.js +45 -88
  161. package/dist/observability/experience/report-structure.d.ts +1 -1
  162. package/dist/observability/experience/report-structure.js +2 -1
  163. package/dist/observability/experience/report-value-guards.d.ts +8 -21
  164. package/dist/observability/experience/report-value-guards.js +230 -1003
  165. package/dist/observability/experience/review-checklist.d.ts +1 -0
  166. package/dist/observability/experience/review-checklist.js +9 -5
  167. package/dist/observability/experience/reviewer-report.d.ts +1 -2
  168. package/dist/observability/experience/reviewer-report.js +2 -2
  169. package/dist/observability/experience/session-story.js +0 -2
  170. package/dist/observability/experience.d.ts +19 -0
  171. package/dist/observability/experience.js +8 -3
  172. package/dist/observability/inbox/effective-review.d.ts +25 -0
  173. package/dist/observability/inbox/effective-review.js +141 -0
  174. package/dist/observability/inbox/index.d.ts +3 -4
  175. package/dist/observability/inbox/index.js +5 -43
  176. package/dist/observability/inbox/resolved-review.d.ts +0 -1
  177. package/dist/observability/inbox/resolved-review.js +3 -38
  178. package/dist/observability/inbox/session-time-range.d.ts +2 -0
  179. package/dist/observability/inbox/session-time-range.js +10 -0
  180. package/dist/observability/inbox/view-model.d.ts +6 -3
  181. package/dist/observability/inbox/view-model.js +8 -24
  182. package/dist/observability/skill-health/analyzer.d.ts +8 -9
  183. package/dist/observability/skill-health/analyzer.js +22 -33
  184. package/dist/observability/soft-standards/index.d.ts +1 -12
  185. package/dist/observability/soft-standards/index.js +1 -12
  186. package/dist/observability/soft-standards/skill-standards-store.d.ts +1 -2
  187. package/dist/observability/soft-standards/skill-standards-store.js +0 -22
  188. package/dist/observability/trace/attribution.d.ts +0 -33
  189. package/dist/observability/trace/attribution.js +0 -92
  190. package/dist/observability/trace/index.d.ts +4 -4
  191. package/dist/observability/trace/index.js +3 -3
  192. package/dist/observability/trace/segmentation.d.ts +0 -3
  193. package/dist/observability/trace/segmentation.js +2 -7
  194. package/dist/observability/trace/source.d.ts +1 -32
  195. package/dist/observability/trace/source.js +2 -36
  196. package/dist/observability/trace/trace-ir.d.ts +0 -2
  197. package/dist/observability/trace/trace-ir.js +0 -6
  198. package/dist/observability/view-models/conversation.d.ts +2 -2
  199. package/dist/observability/view-models/knowledge-debugger.d.ts +2 -2
  200. package/dist/shared/atomic-json.d.ts +0 -4
  201. package/dist/shared/atomic-json.js +0 -15
  202. package/dist/studio/application/skill-index.d.ts +1 -2
  203. package/dist/studio/application/skill-index.js +4 -16
  204. package/dist/studio/application/skill-insights.d.ts +1 -2
  205. package/dist/studio/application/skill-insights.js +0 -13
  206. package/dist/studio/http/routes/observations.js +6 -0
  207. package/dist/studio/presentation/layout.d.ts +0 -3
  208. package/dist/studio/presentation/layout.js +0 -18
  209. package/dist/studio/presentation/observation-inbox/experience-workspace-renderer.js +22 -102
  210. package/dist/studio/presentation/observation-inbox/metric-renderer.d.ts +124 -11
  211. package/dist/studio/presentation/observation-inbox/metric-renderer.js +18 -377
  212. package/dist/studio/presentation/observation-inbox/process-workspace-renderer.d.ts +40 -2
  213. package/dist/studio/presentation/observation-inbox/review-renderer.js +0 -32
  214. package/dist/studio/presentation/observation-inbox-renderer.js +2 -2
  215. package/dist/studio/presentation/skill-detail-renderer.js +4 -13
  216. package/dist/studio/view-models/skill-index.d.ts +2 -0
  217. package/package.json +2 -3
  218. package/dist/cli/lib/parse-run-config/variant-resolution.d.ts +0 -23
  219. package/dist/cli/lib/parse-run-config/variant-resolution.js +0 -98
  220. package/dist/cli/lib/parse-run-config.d.ts +0 -93
  221. package/dist/cli/lib/parse-run-config.js +0 -163
  222. package/dist/eval-workflows/hosts/adapters/index.d.ts +0 -8
  223. package/dist/eval-workflows/hosts/adapters/index.js +0 -8
  224. package/dist/eval-workflows/hosts/evaluators/index.d.ts +0 -6
  225. package/dist/eval-workflows/hosts/evaluators/index.js +0 -6
  226. package/dist/eval-workflows/hosts/index.d.ts +0 -11
  227. package/dist/eval-workflows/hosts/index.js +0 -11
  228. package/dist/eval-workflows/hosts/resource-leases/index.d.ts +0 -3
  229. package/dist/eval-workflows/hosts/resource-leases/index.js +0 -3
  230. package/dist/eval-workflows/inputs/contracts/index.d.ts +0 -5
  231. package/dist/eval-workflows/inputs/contracts/index.js +0 -1
  232. package/dist/eval-workflows/instruments/contracts/index.d.ts +0 -1
  233. package/dist/eval-workflows/instruments/contracts/index.js +0 -1
  234. package/dist/eval-workflows/measurement/analysis/index.d.ts +0 -23
  235. package/dist/eval-workflows/measurement/analysis/index.js +0 -23
  236. package/dist/eval-workflows/measurement/evaluators/index.d.ts +0 -4
  237. package/dist/eval-workflows/measurement/evaluators/index.js +0 -4
  238. package/dist/eval-workflows/orchestration/index.d.ts +0 -5
  239. package/dist/eval-workflows/orchestration/index.js +0 -5
  240. package/dist/executors/contracts/index.d.ts +0 -4
  241. package/dist/executors/contracts/index.js +0 -1
  242. package/dist/knowledge-artifacts/doctor/preflight.d.ts +0 -32
  243. package/dist/knowledge-artifacts/doctor/preflight.js +0 -31
  244. package/dist/observability/contracts/index.d.ts +0 -7
  245. package/dist/observability/contracts/index.js +0 -1
  246. package/dist/observability/conversation/view-model.d.ts +0 -5
  247. package/dist/observability/conversation/view-model.js +0 -100
@@ -73,7 +73,7 @@ omk eval [flags]
73
73
 
74
74
  **Flags:**
75
75
 
76
- - `--batch` `boolean`:batch 模式:baseline 作为对照组,逐个 skill 作为实验组
76
+ - `--batch` `boolean`:批量评测:baseline 作为对照组,逐个 skill 作为实验组;不支持 repeat 大于 1
77
77
  - `--bootstrap` `boolean`:加 bootstrap CI
78
78
  - `--bootstrap-samples` `option`:bootstrap 重采样次数,默认 1000
79
79
  - `--budget-per-sample-ms` `option`:单用例时长上限 ms(必须 > 0,不传则无上限)
@@ -95,7 +95,6 @@ omk eval [flags]
95
95
  - `--layered-stats` `boolean`:输出分层统计
96
96
  - `--mcp-config` `option`:MCP 配置文件路径
97
97
  - `--model` `option`:被测模型
98
- - `--no-cache` `boolean`:跳过 executor cache
99
98
  - `--no-debias-length` `boolean`:关 length-debias(默认开)
100
99
  - `--no-diagnostic` `boolean`:关闭基于 Core 失败、缺失、排除与稳定 reason code 的诊断投影。
101
100
  - `--no-evidence` `boolean`:不把本次评测写成证据追加进受管记录(默认会为已 install 的 skill 自动写)。
@@ -1,11 +1,5 @@
1
- // oclif 版 eval gold topic shim — 裸 `omk eval gold` 走 oclif Help class 打 topic
2
- // help,然后 exit 1(跟 legacy CliExit(1) 行为对齐:CI 脚本靠 exit code 区分「漏写
3
- // sub-sub」)。
4
- //
5
- // 不再依赖 legacy hand-written usage() 字符串(已删):oclif 默认没内建 help command,
6
- // 但 helpClass 是可实例化的 Help 子类。直接 new HelpClass(config).showCommandHelp(
7
- // config.findCommand('eval:gold')) 让 LangAwareHelp 按 --lang 渲染当前 topic 的
8
- // description + sub-sub 列表(init / validate / compare)。
1
+ // 裸 `omk eval gold` 显示帮助并退出 1,让自动化能识别缺少子命令。
2
+ // 复用当前 HelpClass,保持语言选择和子命令列表与 CLI 一致。
9
3
  import { BaseCommand } from '../../../oclif/base-command.js';
10
4
  import { LANG_FLAG, bilingual } from '../../../oclif/i18n.js';
11
5
  export default class EvalGold extends BaseCommand {
@@ -1,7 +1,15 @@
1
1
  import { BaseCommand } from '../../oclif/base-command.js';
2
2
  import { type CliLang } from '../../lib/i18n.js';
3
- import { type RunConfig } from '../../lib/parse-run-config.js';
4
- export declare function formatConnectivityFailureHint(message: string, config: Pick<RunConfig, 'executorName' | 'model' | 'judgeModels' | 'noJudge'>, lang: CliLang, env?: NodeJS.ProcessEnv): string;
3
+ interface ConnectivityFailureContext {
4
+ executorName: string;
5
+ model: string;
6
+ judgeModels: readonly {
7
+ executor: string;
8
+ model: string;
9
+ }[];
10
+ noJudge: boolean;
11
+ }
12
+ export declare function formatConnectivityFailureHint(message: string, config: Pick<ConnectivityFailureContext, 'executorName' | 'model' | 'judgeModels' | 'noJudge'>, lang: CliLang, env?: NodeJS.ProcessEnv): string;
5
13
  export default class Eval extends BaseCommand {
6
14
  static description: string;
7
15
  static examples: {
@@ -23,7 +31,6 @@ export default class Eval extends BaseCommand {
23
31
  'output-dir': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
24
32
  global: import("@oclif/core/interfaces").BooleanFlag<boolean>;
25
33
  'no-judge': import("@oclif/core/interfaces").BooleanFlag<boolean>;
26
- 'no-cache': import("@oclif/core/interfaces").BooleanFlag<boolean>;
27
34
  'dry-run': import("@oclif/core/interfaces").BooleanFlag<boolean>;
28
35
  concurrency: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
29
36
  timeout: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
@@ -58,3 +65,4 @@ export default class Eval extends BaseCommand {
58
65
  };
59
66
  run(): Promise<void>;
60
67
  }
68
+ export {};
@@ -1,12 +1,13 @@
1
1
  import { existsSync } from 'node:fs';
2
- import { join, relative, sep } from 'node:path';
2
+ import { join, relative, resolve, sep } from 'node:path';
3
3
  import { Flags } from '@oclif/core';
4
4
  import { LANG_FLAG, bilingual } from '../../oclif/i18n.js';
5
5
  import { BaseCommand } from '../../oclif/base-command.js';
6
6
  import { enumStringParser, integerStringParser, numberStringParser } from '../../oclif/parsers.js';
7
7
  import { CliExit } from '../../lib/cli-exit.js';
8
8
  import { tCli } from '../../lib/i18n.js';
9
- import { parseRunConfig } from '../../lib/parse-run-config.js';
9
+ import { prepareCliEvaluation } from '../../lib/prepare-evaluation.js';
10
+ import { CliEvaluationInputError } from '../../../eval-workflows/hosts/application.js';
10
11
  import { codexModelFlagValue, codexModelHint } from '../../lib/codex-model-hint.js';
11
12
  import { looksLikeModelUnavailableFailure } from '../../lib/llm-failure-classifier.js';
12
13
  import { DEFAULT_EVALUATION_GATE_THRESHOLD as DEFAULT_GATE_THRESHOLD } from '../../../eval-workflows/evaluation-defaults.js';
@@ -15,7 +16,7 @@ import { shellQuoteArg } from '../../../shared/shell-quote.js';
15
16
  import { executorNamesForFamily } from '../../../executors/core/registry.js';
16
17
  import { DEFAULT_EVALUATION_TIMEOUT_MS } from '../../../eval-workflows/evaluation-defaults.js';
17
18
  // oclif 版 eval(默认 = run 模式) — 单次 typed parse 之后业务 inline。flag schema
18
- // 镜像 RUN_OPTIONS + eval-runner extra = 41 flag。具体语义跟约束在 parseRunConfig 里。
19
+ // 参数语义与约束由 Workflow 的 parseCliEvaluationRequest 维护。
19
20
  //
20
21
  // `omk eval gold ...` 由 src/cli/commands/eval/gold/{init,validate,compare}.ts
21
22
  // 处理,oclif 文件目录路由自动接管,不进 eval.ts。
@@ -127,15 +128,29 @@ export function formatConnectivityFailureHint(message, config, lang, env = proce
127
128
  return '';
128
129
  }
129
130
  async function runEval(_args, flags, lang) {
130
- const { values, config, evalConfig } = parseRunConfig({ ...flags }, { lang });
131
- if (!values.batch && !hasUsableSamplesPath(config.samplesPath)) {
131
+ let prepared;
132
+ try {
133
+ prepared = prepareCliEvaluation({ ...flags }, { lang });
134
+ }
135
+ catch (error) {
136
+ if (!(error instanceof CliEvaluationInputError))
137
+ throw error;
138
+ console.error(`error: ${error.message}`);
139
+ throw new CliExit(2);
140
+ }
141
+ const { request } = prepared;
142
+ const values = flags;
143
+ const evalConfig = prepared.parseInput.evalConfig;
144
+ const samplesPath = resolve(prepared.projectRoot, request.values.locators.samples);
145
+ const skillDir = resolve(prepared.projectRoot, request.values.locators.skillDirectory);
146
+ if (!values.batch && !hasUsableSamplesPath(samplesPath)) {
132
147
  const treatmentRaw = typeof values.treatment === 'string' ? values.treatment : '';
133
148
  const treatments = treatmentRaw.split(',').map((v) => v.trim()).filter(Boolean);
134
149
  const sampleCommand = !values.samples && !evalConfig?.samples && treatments.length === 1
135
- ? sampleCommandForSingleTreatment(treatments[0], config.skillDir)
150
+ ? sampleCommandForSingleTreatment(treatments[0], skillDir)
136
151
  : null;
137
152
  const missingSamplesMessage = [
138
- tCli('cli.common.samples_not_found', lang, { path: config.samplesPath }),
153
+ tCli('cli.common.samples_not_found', lang, { path: samplesPath }),
139
154
  sampleCommand ? tCli('cli.common.samples_not_found_hint', lang, { command: sampleCommand }) : '',
140
155
  ].filter(Boolean).join('\n');
141
156
  console.error(tCli('cli.common.error_prefix', lang, {
@@ -145,12 +160,7 @@ async function runEval(_args, flags, lang) {
145
160
  }
146
161
  try {
147
162
  const { runCoreEvaluationCommand } = await import('../../lib/run-core-evaluation.js');
148
- const result = await runCoreEvaluationCommand({
149
- flags: { ...flags },
150
- config,
151
- evalConfig,
152
- lang,
153
- });
163
+ const result = await runCoreEvaluationCommand({ prepared });
154
164
  if (!process.stdout.isTTY || result.output && typeof result.output === 'object'
155
165
  && result.output.projectionKind === 'core-cli-dry-run') {
156
166
  console.log(JSON.stringify(result.output, null, 2));
@@ -171,7 +181,12 @@ async function runEval(_args, flags, lang) {
171
181
  throw err;
172
182
  const message = err instanceof Error ? err.message : String(err);
173
183
  console.error(tCli('cli.common.error_prefix', lang, {
174
- message: `${message}${formatConnectivityFailureHint(message, config, lang)}`,
184
+ message: `${message}${formatConnectivityFailureHint(message, {
185
+ executorName: request.values.targetRuntime.executorId,
186
+ model: request.values.targetRuntime.model,
187
+ noJudge: !request.values.judges.enabled,
188
+ judgeModels: request.values.judges.members.map((member) => ({ executor: member.executorId, model: member.model })),
189
+ }, lang, prepared.environment.environment)}`,
175
190
  }));
176
191
  throw new CliExit(1);
177
192
  }
@@ -262,9 +277,6 @@ export default class Eval extends BaseCommand {
262
277
  'no-judge': Flags.boolean({
263
278
  description: bilingual({ zh: '跳过 LLM 评委', en: 'Skip LLM judge' }),
264
279
  }),
265
- 'no-cache': Flags.boolean({
266
- description: bilingual({ zh: '跳过 executor cache', en: 'Skip executor cache' }),
267
- }),
268
280
  'dry-run': Flags.boolean({
269
281
  description: bilingual({ zh: '只 plan 不实跑', en: 'Plan only, no real exec' }),
270
282
  }),
@@ -281,8 +293,8 @@ export default class Eval extends BaseCommand {
281
293
  }),
282
294
  batch: Flags.boolean({
283
295
  description: bilingual({
284
- zh: 'batch 模式:baseline 作为对照组,逐个 skill 作为实验组',
285
- en: 'Batch mode: baseline vs each skill',
296
+ zh: '批量评测:baseline 作为对照组,逐个 skill 作为实验组;不支持 repeat 大于 1',
297
+ en: 'Batch mode: baseline vs each skill; repeat must be 1',
286
298
  }),
287
299
  }),
288
300
  'skip-connectivity': Flags.boolean({
@@ -92,7 +92,7 @@ export async function runEvolve(args, flags, lang) {
92
92
  samplesFile = resolvedInput.samplesPath;
93
93
  }
94
94
  // 参数校验必须早于任何昂贵副作用(自动生成用例 / LLM 调用)。
95
- const { parseJudgeModelsArgOrExit } = await import('../lib/parse-run-config.js');
95
+ const { parseJudgeModelsArgOrExit } = await import('../lib/parse-run-config/judge-models.js');
96
96
  const evolveJudges = parseJudgeModelsArgOrExit(flags['judge-models']);
97
97
  if (evolveJudges.length > 1) {
98
98
  console.error(tCli('cli.common.judge_models_single_only', lang, { cmd: 'omk evolve' }));
@@ -168,13 +168,15 @@ export async function runEvolve(args, flags, lang) {
168
168
  }
169
169
  const { evolveSkillCore } = await import('../../knowledge-artifacts/authoring/core-evolver.js');
170
170
  const { runCoreEvaluationCommand } = await import('../lib/run-core-evaluation.js');
171
+ const { prepareCliEvaluation } = await import('../lib/prepare-evaluation.js');
171
172
  const evolveEffort = flags.effort ? validateEvolveEffort(flags.effort, lang) : undefined;
172
173
  const evaluatePair = async (control, treatment) => {
173
174
  const evaluation = await runCoreEvaluationCommand({
174
- flags: {
175
+ prepared: prepareCliEvaluation({
175
176
  control,
176
177
  treatment,
177
178
  samples: resolve(samplesFile),
179
+ 'skill-dir': dirname(resolve(skillPath)),
178
180
  executor: flags.executor,
179
181
  model: flags.model,
180
182
  'judge-models': evolveJudges.map((judge) => `${judge.executor}:${judge.model}`).join(','),
@@ -185,17 +187,7 @@ export async function runEvolve(args, flags, lang) {
185
187
  'no-evidence': true,
186
188
  'no-serve': true,
187
189
  'report-only': true,
188
- },
189
- config: {
190
- samplesPath: resolve(samplesFile),
191
- skillDir: dirname(resolve(skillPath)),
192
- executorName: flags.executor,
193
- model: flags.model,
194
- effort: evolveEffort,
195
- judgeModels: evolveJudges,
196
- },
197
- evalConfig: null,
198
- lang: 'zh',
190
+ }, { lang: 'zh' }),
199
191
  });
200
192
  if (evaluation.stored === undefined) {
201
193
  throw new Error('Core evolve evaluation 未持久化 artifact chain。');
@@ -91,8 +91,7 @@ export default class Init extends BaseCommand {
91
91
  }),
92
92
  required: false,
93
93
  parse: async (input) => {
94
- // 拒绝 `omk init -- --weird` 这种把 flag 当 positional 的写法 — legacy 会
95
- // 创建名为 `--weird` 的目录,新人一头雾水。在 oclif Args 这层拦住更友好。
94
+ // 在参数解析阶段拒绝疑似遗漏的 flag,避免误建名为 `--weird` 的目录。
96
95
  if (input.startsWith('--')) {
97
96
  const lang = resolveLang();
98
97
  const msg = lang === 'zh'
@@ -114,7 +114,9 @@ export async function runObserveInbox(_args, flags, lang) {
114
114
  items = items.slice(0, limit);
115
115
  }
116
116
  if (flags.json) {
117
- console.log(JSON.stringify({ kind: 'observe-inbox-query', items }, null, 2));
117
+ const { buildObservationInboxViewModel } = await import('../../../observability/inbox/view-model.js');
118
+ const { effectiveExperienceReports, resolvedReviewSessions, unappliedMetricAnnotations } = buildObservationInboxViewModel(dir, { skill: flags.skill });
119
+ console.log(JSON.stringify({ kind: 'observe-inbox-query', items, effectiveExperienceReports, resolvedReviewSessions, unappliedMetricAnnotations }, null, 2));
118
120
  return;
119
121
  }
120
122
  if (items.length === 0) {
@@ -1,4 +1,4 @@
1
- import { execFileSync } from 'node:child_process';
1
+ import { resolveActor } from '../lib/actor.js';
2
2
  import { Args, Flags } from '@oclif/core';
3
3
  import { LANG_FLAG, bilingual } from '../oclif/i18n.js';
4
4
  import { BaseCommand } from '../oclif/base-command.js';
@@ -7,18 +7,6 @@ import { CliExit } from '../lib/cli-exit.js';
7
7
  import { sanitizeCell } from '../lib/cell-format.js';
8
8
  import { appendManagedDecision, evaluatePromoteGate, globalManagedDir, isCurrentlyPromoted, loadManagedRecord, probeSourceState, managedDir, managedRecordId, resolveManagedDir, } from '../../knowledge-artifacts/governance/index.js';
9
9
  const SUPPORTED_KINDS = ['skill'];
10
- /** 决定的 actor:--actor > git config user.name > $USER / $LOGNAME > unknown。git 参数无用户输入,安全。 */
11
- function resolveActor(flagActor) {
12
- if (flagActor && flagActor.trim())
13
- return flagActor.trim();
14
- try {
15
- const name = execFileSync('git', ['config', 'user.name'], { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
16
- if (name)
17
- return name;
18
- }
19
- catch { /* git 缺失 / 无配置 → 回退环境 */ }
20
- return process.env.USER || process.env.LOGNAME || 'unknown';
21
- }
22
10
  /** 洗 detail 里来自受管 JSON 的不可信值(verdict / judgePromptHash),文本输出防 ANSI/OSC 注入。 */
23
11
  function safeDetail(detail) {
24
12
  if (!detail)
@@ -1,4 +1,4 @@
1
- import { execFileSync } from 'node:child_process';
1
+ import { resolveActor } from '../lib/actor.js';
2
2
  import { Args, Flags } from '@oclif/core';
3
3
  import { LANG_FLAG, bilingual } from '../oclif/i18n.js';
4
4
  import { BaseCommand } from '../oclif/base-command.js';
@@ -7,18 +7,6 @@ import { CliExit } from '../lib/cli-exit.js';
7
7
  import { sanitizeCell } from '../lib/cell-format.js';
8
8
  import { appendManagedDecision, globalManagedDir, isCurrentlyPromoted, loadManagedRecord, managedDir, managedRecordId, resolveManagedDir, } from '../../knowledge-artifacts/governance/index.js';
9
9
  const SUPPORTED_KINDS = ['skill'];
10
- /** 决定的 actor:--actor > git config user.name > $USER / $LOGNAME > unknown。git 参数无用户输入,安全。 */
11
- function resolveActor(flagActor) {
12
- if (flagActor && flagActor.trim())
13
- return flagActor.trim();
14
- try {
15
- const name = execFileSync('git', ['config', 'user.name'], { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
16
- if (name)
17
- return name;
18
- }
19
- catch { /* git 缺失 / 无配置 → 回退环境 */ }
20
- return process.env.USER || process.env.LOGNAME || 'unknown';
21
- }
22
10
  export default class Rollback extends BaseCommand {
23
11
  static description = bilingual({
24
12
  zh: '回退受管 skill 当前版本的 promoted 接受:撤销最近一次 promote,在记录里追加一条 rollback 决定(源未漂移则状态回到 measurable,源已漂移则仍 stale)。',
@@ -0,0 +1,2 @@
1
+ /** Resolve a decision actor from the flag, Git config, environment, then fallback. */
2
+ export declare function resolveActor(flagActor: string | undefined): string;
@@ -0,0 +1,13 @@
1
+ import { execFileSync } from 'node:child_process';
2
+ /** Resolve a decision actor from the flag, Git config, environment, then fallback. */
3
+ export function resolveActor(flagActor) {
4
+ if (flagActor && flagActor.trim())
5
+ return flagActor.trim();
6
+ try {
7
+ const name = execFileSync('git', ['config', 'user.name'], { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'] }).trim();
8
+ if (name)
9
+ return name;
10
+ }
11
+ catch { /* git 缺失 / 无配置 → 回退环境 */ }
12
+ return process.env.USER || process.env.LOGNAME || 'unknown';
13
+ }
@@ -36,7 +36,7 @@
36
36
  * 优先级 `--lang CLI flag > OMK_LANG env > zh`。
37
37
  *
38
38
  * 三态 boolean 注意:oclif 对 `Flags.boolean({})`(没 `default`)运行时不会塞 `false`,
39
- * 而是字段缺失,字段读出来是 `undefined`。这点对 `eval` 跟 `parseRunConfig()` fallback
39
+ * 而是字段缺失,字段读出来是 `undefined`。这点对 `eval` 跟 `parseCliEvaluationRequest()` fallback
40
40
  * 关键 — `undefined`(CLI 没传)走 `eval.yaml` config / 内置 default;`false`(显式
41
41
  * `--no-xxx`)真的关。如果在 interface 里把它标成 `boolean`,后续维护者按类型读
42
42
  * `flags.bootstrap` 会静默把「没传」当成「关」,绕开 config fallback。所以未设
@@ -80,7 +80,6 @@ export interface EvalFlags {
80
80
  /** 报告写全局而非项目 .omk/eval(写入侧 escape;读取侧 studio / 复用走 overlay 兜底)。 */
81
81
  global?: boolean;
82
82
  'no-judge'?: boolean;
83
- 'no-cache'?: boolean;
84
83
  'dry-run'?: boolean;
85
84
  concurrency?: string;
86
85
  timeout?: string;
@@ -36,7 +36,7 @@
36
36
  * 优先级 `--lang CLI flag > OMK_LANG env > zh`。
37
37
  *
38
38
  * 三态 boolean 注意:oclif 对 `Flags.boolean({})`(没 `default`)运行时不会塞 `false`,
39
- * 而是字段缺失,字段读出来是 `undefined`。这点对 `eval` 跟 `parseRunConfig()` fallback
39
+ * 而是字段缺失,字段读出来是 `undefined`。这点对 `eval` 跟 `parseCliEvaluationRequest()` fallback
40
40
  * 关键 — `undefined`(CLI 没传)走 `eval.yaml` config / 内置 default;`false`(显式
41
41
  * `--no-xxx`)真的关。如果在 interface 里把它标成 `boolean`,后续维护者按类型读
42
42
  * `flags.bootstrap` 会静默把「没传」当成「关」,绕开 config fallback。所以未设
@@ -1,3 +1,3 @@
1
1
  import type { CliMessage } from './types.js';
2
- export type CommonMessageKey = 'cli.common.unknown_domain' | 'cli.common.error_prefix' | 'cli.common.skill_dir_not_found' | 'cli.common.skill_file_not_found' | 'cli.common.skill_dir_no_skill_md' | 'cli.common.report_not_found' | 'cli.common.no_judge_model' | 'cli.common.judge_models_single_only' | 'cli.common.warn_load_samples_failed' | 'cli.common.ambiguous_sample_files' | 'cli.common.samples_not_found' | 'cli.common.samples_not_found_hint' | 'cli.update.new_version_available' | 'cli.update.box_title' | 'cli.update.box_version_line' | 'cli.update.box_upgrade_line' | 'cli.update.box_silence_line' | 'cli.observe.view_hint' | 'cli.observe.observation_recorded' | 'cli.observe.production_gap' | 'cli.studio.started' | 'cli.studio.stop_hint' | 'cli.studio.open_failed' | 'cli.doctor.no_skill_found' | 'cli.doctor.progress_skill_start' | 'cli.doctor.progress_skill_done';
2
+ export type CommonMessageKey = 'cli.common.unknown_domain' | 'cli.common.error_prefix' | 'cli.common.skill_dir_not_found' | 'cli.common.skill_file_not_found' | 'cli.common.skill_dir_no_skill_md' | 'cli.common.judge_models_single_only' | 'cli.common.ambiguous_sample_files' | 'cli.common.samples_not_found' | 'cli.common.samples_not_found_hint' | 'cli.update.new_version_available' | 'cli.update.box_title' | 'cli.update.box_version_line' | 'cli.update.box_upgrade_line' | 'cli.update.box_silence_line' | 'cli.observe.view_hint' | 'cli.observe.observation_recorded' | 'cli.observe.production_gap' | 'cli.studio.started' | 'cli.studio.stop_hint' | 'cli.studio.open_failed' | 'cli.doctor.no_skill_found' | 'cli.doctor.progress_skill_start' | 'cli.doctor.progress_skill_done';
3
3
  export declare const commonDict: Record<CommonMessageKey, CliMessage>;
@@ -19,22 +19,10 @@ export const commonDict = {
19
19
  zh: '目录下未找到 SKILL.md: {path}',
20
20
  en: 'SKILL.md not found in directory: {path}',
21
21
  },
22
- 'cli.common.report_not_found': {
23
- zh: '未找到 report: {id}',
24
- en: 'Report not found: {id}',
25
- },
26
- 'cli.common.no_judge_model': {
27
- zh: '未指定评委。请加 --judge-models <executor:model>, 或确保 report.meta.judgeModels 已写。',
28
- en: 'No judge configured. Pass --judge-models <executor:model> or ensure the report has meta.judgeModels.',
29
- },
30
22
  'cli.common.judge_models_single_only': {
31
23
  zh: '{cmd} 仅支持单评委。--judge-models 只能传一个 executor:model entry。',
32
24
  en: '{cmd} only supports a single judge. --judge-models accepts exactly one executor:model entry.',
33
25
  },
34
- 'cli.common.warn_load_samples_failed': {
35
- zh: '⚠ 加载 samples 文件失败 ({path}): {message}\n',
36
- en: '⚠ Failed to load samples file ({path}): {message}\n',
37
- },
38
26
  'cli.common.ambiguous_sample_files': {
39
27
  zh: '评测用例文件存在歧义:{paths}。同一作用域只能保留 eval-samples.json 或 eval-samples.yaml 其中一个。',
40
28
  en: 'Ambiguous eval sample files: {paths}. Keep only one of eval-samples.json or eval-samples.yaml in the same scope.',
@@ -1,3 +1,3 @@
1
1
  import type { CliMessage } from './types.js';
2
- export type HelpMessageKey = 'cli.help.observe' | 'cli.help.observe_ingest' | 'cli.help.observe_inbox' | 'cli.help.observe_show' | 'cli.help.evolve' | 'cli.help.sample' | 'cli.help.studio';
2
+ export type HelpMessageKey = 'cli.help.observe';
3
3
  export declare const helpDict: Record<HelpMessageKey, CliMessage>;
@@ -33,232 +33,6 @@ Options:
33
33
  --global Write to global ~/.oh-my-knowledge/observe/health instead of project .omk/observe/health
34
34
 
35
35
  The observe inbox is a separate line; see omk observe inbox --help.
36
- `,
37
- },
38
- 'cli.help.observe_ingest': {
39
- zh: `
40
- omk observe ingest — 读取真实 session trace,写入 observe inbox 数据
41
-
42
- 用法:
43
- omk observe ingest <sessions-dir-or-file> [options]
44
-
45
- 选项:
46
- --output-dir <path> 输出目录(默认:.omk/observe/inbox;读取时兜底到 ~/.oh-my-knowledge/observe/inbox)
47
-
48
- 支持:
49
- Claude Code JSONL
50
- Markdown 对话日志(.log)
51
- `,
52
- en: `
53
- omk observe ingest — read real session traces and write observe inbox data
54
-
55
- Usage:
56
- omk observe ingest <sessions-dir-or-file> [options]
57
-
58
- Options:
59
- --output-dir <path> Output directory (default: .omk/observe/inbox; read fallback: ~/.oh-my-knowledge/observe/inbox)
60
-
61
- Supported:
62
- Claude Code JSONL
63
- Markdown conversation logs (.log)
64
- `,
65
- },
66
- 'cli.help.observe_inbox': {
67
- zh: `
68
- omk observe inbox — 查看已写入的 observe inbox 问题列表
69
-
70
- 用法:
71
- omk observe inbox [options]
72
-
73
- 选项:
74
- --input-dir <path> 读取目录(默认:.omk/observe/inbox;兜底到 ~/.oh-my-knowledge/observe/inbox)
75
- --limit <n> 展示 top N(默认:20)
76
- --skill <name> 只看指定 skill
77
- --explore <n> 从最近 50 条 medium/low 问题里抽样查看长尾
78
- --include-noise --explore 时显式包含 noise 桶
79
- --by-skill 按 skill 输出资产看板
80
- --llm-enhanced-review 显式调用模型进行链路增强复盘
81
- --model <name> LLM 增强复盘使用的模型(默认:sonnet)
82
- --executor <name> LLM 增强复盘使用的执行器
83
- --refresh 强制重新生成 LLM 增强复盘
84
- --json 输出 JSON
85
- `,
86
- en: `
87
- omk observe inbox — inspect previously ingested observe inbox items
88
-
89
- Usage:
90
- omk observe inbox [options]
91
-
92
- Options:
93
- --input-dir <path> Input directory (default: .omk/observe/inbox; fallback: ~/.oh-my-knowledge/observe/inbox)
94
- --limit <n> Show top N (default: 20)
95
- --skill <name> Only show one skill
96
- --explore <n> Sample long-tail issues from the latest 50 medium/low items
97
- --include-noise Explicitly include the noise bucket with --explore
98
- --by-skill Print the skill-level asset board
99
- --llm-enhanced-review Explicitly run model-based enhanced chain review
100
- --model <name> Model for LLM enhanced review (default: sonnet)
101
- --executor <name> Executor for LLM enhanced review
102
- --refresh Force LLM enhanced review refresh
103
- --json Print JSON
104
- `,
105
- },
106
- 'cli.help.observe_show': {
107
- zh: `
108
- omk observe show — 查看单条 observation 的原始上下文
109
-
110
- 用法:
111
- omk observe show <inbox_id> [options]
112
-
113
- 选项:
114
- --input-dir <path> 读取目录(默认:.omk/observe/inbox;兜底到 ~/.oh-my-knowledge/observe/inbox)
115
- `,
116
- en: `
117
- omk observe show — inspect the raw context around one observation
118
-
119
- Usage:
120
- omk observe show <inbox_id> [options]
121
-
122
- Options:
123
- --input-dir <path> Input directory (default: .omk/observe/inbox; fallback: ~/.oh-my-knowledge/observe/inbox)
124
- `,
125
- },
126
- 'cli.help.evolve': {
127
- zh: `
128
- omk evolve——多轮自动迭代改进 skill
129
-
130
- 用法:
131
- omk evolve <skill-path> [options]
132
-
133
- 选项:
134
- --rounds <n> 迭代轮数(默认:5)
135
- --target <score> 目标分数
136
- --model <name> 任务执行模型,默认跟随 runtime;Codex 读取本机配置
137
- --improve-model <name> skill 改写模型,默认沿用任务执行模型
138
- --judge-models <executor:model> 单评委配置(默认跟随执行器;Codex 沿用被测模型)
139
-
140
- 示例:
141
- omk evolve skills/code-review/SKILL.md
142
- omk evolve skills/code-review/SKILL.md --rounds 10 --target 4.5
143
- omk evolve skills/code-review/SKILL.md --model sonnet --improve-model opus
144
- `,
145
- en: `
146
- omk evolve — auto-iterate a skill through multi-round evaluation loops
147
-
148
- Usage:
149
- omk evolve <skill-path> [options]
150
-
151
- Options:
152
- --rounds <n> Iteration rounds (default: 5)
153
- --target <score> Target score
154
- --model <name> Task executor model; follows runtime (Codex reads local config)
155
- --improve-model <name> Skill rewriter model; defaults to the task model
156
- --judge-models <executor:model> Single judge config (follows executor; Codex reuses task model)
157
-
158
- Examples:
159
- omk evolve skills/code-review/SKILL.md
160
- omk evolve skills/code-review/SKILL.md --rounds 10 --target 4.5
161
- omk evolve skills/code-review/SKILL.md --model sonnet --improve-model opus
162
- `,
163
- },
164
- 'cli.help.sample': {
165
- zh: `
166
- omk sample——生成或补齐 eval-samples 评测用例
167
-
168
- 用法:
169
- omk sample <skill-path> [options]
170
- omk sample --batch [--skill-dir <dir>] [options]
171
-
172
- 输出位置(默认):
173
- 目录 skill(<skill>/SKILL.md) → <skill>/.omk/eval-samples.json(也支持 .yaml)
174
- 扁平 .md(单次) → 当前目录的 eval-samples.json(项目级兜底)
175
- 扁平 .md(--batch) → 跳过;私有用例请迁移为目录 skill
176
-
177
- 选项:
178
- --count <n> 强制生成 N 条(不指定时由 LLM 按 skill 类型自动判断:
179
- 工作流型 6-8 条 / 原子型 4-6 条 / 混合型 5-7 条)
180
- --model <name> 生成模型(默认:opus;lean+effort-low 已自动开,想省钱可改 sonnet/haiku)
181
- --focus <text> 自然语言指定希望覆盖的场景(追加到 prompt,优先级高于自由发挥)
182
- --batch 为 skill 目录下缺少 samples 的 skill 批量生成
183
- --skill-dir <path> skill 目录(batch 使用,默认:skills)
184
-
185
- 示例:
186
- omk sample skills/req-tool.md
187
- omk sample skills/req-tool.md --count 8 \\
188
- --focus "重点覆盖 tag 查询走 PROJECT 空 → WORKSPACE 兜底的多步流程,以及 search 失败的错误路径"
189
- `,
190
- en: `
191
- omk sample — generate or fill eval-samples test cases
192
-
193
- Usage:
194
- omk sample <skill-path> [options]
195
- omk sample --batch [--skill-dir <dir>] [options]
196
-
197
- Output path (default):
198
- directory skill (<skill>/SKILL.md) → <skill>/.omk/eval-samples.json (.yaml also supported)
199
- flat .md (single) → ./eval-samples.json in current directory (project fallback)
200
- flat .md (--batch) → skipped; migrate to a directory skill for private samples
201
-
202
- Options:
203
- --count <n> Force N samples (omit to let LLM auto-decide by skill type:
204
- workflow 6-8 / atomic 4-6 / mixed 5-7)
205
- --model <name> Generation model (default: opus; lean+effort-low applied; pass --model sonnet/haiku to save cost)
206
- --focus <text> Natural-language scenario hints appended to the prompt (overrides freeform diversity)
207
- --batch Generate for skills that are missing samples
208
- --skill-dir <path> Skill directory for batch mode (default: skills)
209
-
210
- Examples:
211
- omk sample skills/req-tool.md
212
- omk sample skills/req-tool.md --count 8 \\
213
- --focus "Cover PROJECT-empty → WORKSPACE-fallback multi-step tag lookup and the search-failure error path"
214
- `,
215
- },
216
- 'cli.help.studio': {
217
- zh: `
218
- omk studio——打开本地知识工作台
219
-
220
- 用法:
221
- omk studio [options]
222
-
223
- 选项:
224
- --port <n> 本地服务端口(默认:7799)
225
- --host <host> 监听地址(默认:127.0.0.1;局域网访问可用 0.0.0.0)
226
- --reports-dir <path> 只看指定 Core 报告目录(默认聚合当前项目 + 全局)
227
- --analyses-dir <path> 观测健康报告目录(默认:项目级 .omk/observe/health,空则全局兜底)
228
- --doctors-dir <path> 体检报告目录(默认:项目级 .omk/doctor,空则全局兜底)
229
- --observations-dir <path> observe inbox 数据目录(默认:.omk/observe/inbox)
230
- --global 只看全局 eval / observe/health / doctor / observe/inbox 目录(~/.oh-my-knowledge/),而非机器级聚合 / 项目优先;governance/managed 不受影响
231
- --no-open 只启动服务,不自动打开浏览器
232
- --dev 开发模式:文件变化时自动重启
233
-
234
- 示例:
235
- omk studio
236
- omk studio --port 7798
237
- omk studio --host 0.0.0.0 --observations-dir .omk/observe/inbox
238
- omk studio --no-open
239
- `,
240
- en: `
241
- omk studio — open the local knowledge workbench
242
-
243
- Usage:
244
- omk studio [options]
245
-
246
- Options:
247
- --port <n> Local server port (default: 7799)
248
- --host <host> Listen address (default: 127.0.0.1; use 0.0.0.0 for LAN access)
249
- --reports-dir <path> View only this Core reports dir (default aggregates current project + global)
250
- --analyses-dir <path> Observe-health reports dir (default: project .omk/observe/health, falls back to global)
251
- --doctors-dir <path> Doctor reports dir (default: project .omk/doctor, falls back to global)
252
- --observations-dir <path> Observe inbox data directory (default: .omk/observe/inbox)
253
- --global View only global eval, observe/health, doctor, and observe/inbox directories under ~/.oh-my-knowledge/; does not affect governance/managed
254
- --no-open Start the server without opening a browser
255
- --dev Dev mode: restart on file changes
256
-
257
- Examples:
258
- omk studio
259
- omk studio --port 7798
260
- omk studio --host 0.0.0.0 --observations-dir .omk/observe/inbox
261
- omk studio --no-open
262
36
  `,
263
37
  },
264
38
  };
@@ -1,3 +1,3 @@
1
1
  import type { CliMessage } from './types.js';
2
- export type RunMessageKey = 'cli.progress.preflight_starting' | 'cli.progress.sample_retry' | 'cli.progress.sample_error' | 'cli.progress.sample_executing' | 'cli.progress.sample_exec_done' | 'cli.progress.output_preview' | 'cli.progress.judging' | 'cli.progress.judged' | 'cli.progress.skipped' | 'cli.progress.sample_done' | 'cli.progress.sample_failed_done' | 'cli.run.invalid_repeat' | 'cli.run.invalid_holdout_ratio' | 'cli.run.invalid_judge_repeat' | 'cli.run.no_debias_length_active' | 'cli.run.invalid_bootstrap_samples' | 'cli.run.bootstrap_samples_too_large' | 'cli.run.dry_run_no_scores' | 'cli.run.skill_section' | 'cli.run.run_section' | 'cli.run.batch_complete' | 'cli.run.batch_verdict_header' | 'cli.run.batch_verdict_next_step' | 'cli.run.batch_child_report_missing' | 'cli.run.eval_complete' | 'cli.run.tally' | 'cli.run.report_saved' | 'cli.run.evidence_recorded' | 'cli.run.evidence_recorded_promotable' | 'cli.run.evidence_recorded_unbound' | 'cli.run.report_only_gate_skipped' | 'cli.run.report_server_running' | 'cli.run.report_server_view' | 'cli.run.report_server_stop' | 'cli.run.no_serve_in_non_tty' | 'cli.run.no_serve_view_hint' | 'cli.run.gold_load_failed' | 'cli.run.gold_load_issue' | 'cli.run.contamination_warning' | 'cli.run.codex_fallback_hint' | 'cli.run.codex_auth_hint' | 'cli.run.codex_model_hint' | 'cli.run.openai_api_auth_hint' | 'cli.run.openai_api_model_hint' | 'cli.run.anthropic_api_auth_hint' | 'cli.run.anthropic_api_model_hint' | 'cli.run.skip_connectivity_warning';
2
+ export type RunMessageKey = 'cli.run.batch_verdict_header' | 'cli.run.codex_fallback_hint' | 'cli.run.codex_auth_hint' | 'cli.run.codex_model_hint' | 'cli.run.openai_api_auth_hint' | 'cli.run.openai_api_model_hint' | 'cli.run.anthropic_api_auth_hint' | 'cli.run.anthropic_api_model_hint';
3
3
  export declare const runDict: Record<RunMessageKey, CliMessage>;