oh-my-knowledge 0.48.0 → 0.49.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (243) hide show
  1. package/README.md +50 -18
  2. package/README.zh.md +55 -23
  3. package/dist/analysis/coverage-analyzer.d.ts +1 -0
  4. package/dist/analysis/coverage-analyzer.js +125 -62
  5. package/dist/analysis/failure-clusterer.js +2 -1
  6. package/dist/analysis/gap-analyzer.d.ts +2 -2
  7. package/dist/analysis/gap-analyzer.js +13 -3
  8. package/dist/analysis/hedging-classifier.d.ts +2 -2
  9. package/dist/analysis/hedging-classifier.js +3 -4
  10. package/dist/analysis/report-diagnostics.js +9 -7
  11. package/dist/analysis/sample-diagnostics.js +6 -6
  12. package/dist/artifact-graph/doctor.js +15 -7
  13. package/dist/assets/agent-skills/omk/SKILL.md +27 -7
  14. package/dist/assets/agent-skills/omk/references/commands.md +18 -17
  15. package/dist/authoring/evolver.d.ts +10 -6
  16. package/dist/authoring/evolver.js +496 -83
  17. package/dist/authoring/generator.d.ts +3 -3
  18. package/dist/authoring/generator.js +5 -10
  19. package/dist/authoring/sample-fixer.d.ts +8 -6
  20. package/dist/authoring/sample-fixer.js +76 -5
  21. package/dist/cli/commands/doctor.js +31 -14
  22. package/dist/cli/commands/eval/index.d.ts +3 -0
  23. package/dist/cli/commands/eval/index.js +163 -17
  24. package/dist/cli/commands/evolve.d.ts +4 -4
  25. package/dist/cli/commands/evolve.js +27 -13
  26. package/dist/cli/commands/init.js +16 -3
  27. package/dist/cli/commands/observe/inbox.js +28 -21
  28. package/dist/cli/commands/observe/index.js +20 -11
  29. package/dist/cli/commands/observe/ingest.d.ts +3 -0
  30. package/dist/cli/commands/observe/ingest.js +30 -2
  31. package/dist/cli/commands/sample.d.ts +6 -3
  32. package/dist/cli/commands/sample.js +72 -68
  33. package/dist/cli/lib/codex-model-hint.d.ts +9 -0
  34. package/dist/cli/lib/codex-model-hint.js +45 -0
  35. package/dist/cli/lib/generation-failure-hint.d.ts +2 -0
  36. package/dist/cli/lib/generation-failure-hint.js +61 -0
  37. package/dist/cli/lib/i18n-dict/common.d.ts +1 -1
  38. package/dist/cli/lib/i18n-dict/common.js +4 -0
  39. package/dist/cli/lib/i18n-dict/gen.d.ts +1 -1
  40. package/dist/cli/lib/i18n-dict/gen.js +38 -6
  41. package/dist/cli/lib/i18n-dict/help.js +6 -6
  42. package/dist/cli/lib/i18n-dict/init.d.ts +1 -1
  43. package/dist/cli/lib/i18n-dict/init.js +13 -9
  44. package/dist/cli/lib/i18n-dict/run.d.ts +1 -1
  45. package/dist/cli/lib/i18n-dict/run.js +34 -2
  46. package/dist/cli/lib/llm-failure-classifier.d.ts +2 -0
  47. package/dist/cli/lib/llm-failure-classifier.js +8 -0
  48. package/dist/cli/lib/parse-run-config.d.ts +6 -5
  49. package/dist/cli/lib/parse-run-config.js +16 -9
  50. package/dist/cli/lib/runtime-defaults.d.ts +21 -0
  51. package/dist/cli/lib/runtime-defaults.js +79 -0
  52. package/dist/diagnosis/observe-mapper.js +14 -15
  53. package/dist/diagnosis/observe-producer.js +3 -1
  54. package/dist/diagnosis/studio-projection.js +14 -7
  55. package/dist/diagnosis/types.d.ts +2 -0
  56. package/dist/diagnosis/types.js +12 -0
  57. package/dist/doctor/endpoint-rule.js +2 -1
  58. package/dist/eval-core/artifact-file-names.js +18 -1
  59. package/dist/eval-core/artifact-index.d.ts +7 -11
  60. package/dist/eval-core/artifact-index.js +139 -80
  61. package/dist/eval-core/cache.d.ts +12 -3
  62. package/dist/eval-core/cache.js +89 -29
  63. package/dist/eval-core/comparability.js +10 -6
  64. package/dist/eval-core/evaluation-execution.d.ts +2 -1
  65. package/dist/eval-core/evaluation-execution.js +122 -37
  66. package/dist/eval-core/evaluation-job.d.ts +4 -1
  67. package/dist/eval-core/evaluation-job.js +4 -1
  68. package/dist/eval-core/evaluation-reporting.d.ts +15 -13
  69. package/dist/eval-core/evaluation-reporting.js +54 -52
  70. package/dist/eval-core/execution-strategy.d.ts +2 -0
  71. package/dist/eval-core/execution-strategy.js +11 -9
  72. package/dist/eval-core/fact-checker.js +15 -7
  73. package/dist/eval-core/holdout.js +3 -2
  74. package/dist/eval-core/judge-independence.d.ts +2 -2
  75. package/dist/eval-core/mock-hook.cjs +23 -6
  76. package/dist/eval-core/mocks-runtime.js +30 -8
  77. package/dist/eval-core/report-document.d.ts +12 -0
  78. package/dist/eval-core/report-document.js +1151 -0
  79. package/dist/eval-core/report-extensions.d.ts +4 -0
  80. package/dist/eval-core/report-extensions.js +500 -0
  81. package/dist/eval-core/report-file-migration.js +7 -2
  82. package/dist/eval-core/resume-compatibility.d.ts +31 -0
  83. package/dist/eval-core/resume-compatibility.js +141 -0
  84. package/dist/eval-core/sample-fingerprint.d.ts +12 -0
  85. package/dist/eval-core/sample-fingerprint.js +193 -0
  86. package/dist/eval-core/schema.js +86 -31
  87. package/dist/eval-core/verdict.d.ts +8 -4
  88. package/dist/eval-core/verdict.js +24 -10
  89. package/dist/eval-workflows/batch-evaluation-workflow.d.ts +2 -1
  90. package/dist/eval-workflows/batch-evaluation-workflow.js +25 -12
  91. package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.d.ts +10 -5
  92. package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.js +58 -21
  93. package/dist/eval-workflows/evaluation-pipeline/report-finalize.js +3 -1
  94. package/dist/eval-workflows/evaluation-pipeline/run-state.d.ts +4 -1
  95. package/dist/eval-workflows/evaluation-pipeline/run-state.js +4 -1
  96. package/dist/eval-workflows/evaluation-pipeline/test-set-hash.d.ts +6 -5
  97. package/dist/eval-workflows/evaluation-pipeline/test-set-hash.js +17 -10
  98. package/dist/eval-workflows/evaluation-pipeline.js +12 -7
  99. package/dist/eval-workflows/run-evaluation.d.ts +9 -7
  100. package/dist/eval-workflows/run-evaluation.js +79 -51
  101. package/dist/executors/anthropic-api.js +65 -9
  102. package/dist/executors/claude-cli.js +16 -79
  103. package/dist/executors/claude-protocol.d.ts +28 -0
  104. package/dist/executors/claude-protocol.js +180 -0
  105. package/dist/executors/claude-sdk-trace.js +56 -28
  106. package/dist/executors/claude-sdk.d.ts +1 -0
  107. package/dist/executors/claude-sdk.js +39 -93
  108. package/dist/executors/codex-cli-trace.js +166 -31
  109. package/dist/executors/codex-cli.d.ts +6 -8
  110. package/dist/executors/codex-cli.js +49 -151
  111. package/dist/executors/codex-protocol.d.ts +24 -0
  112. package/dist/executors/codex-protocol.js +234 -0
  113. package/dist/executors/codex-sdk.js +68 -120
  114. package/dist/executors/gemini.js +88 -13
  115. package/dist/executors/index.d.ts +2 -3
  116. package/dist/executors/index.js +5 -3
  117. package/dist/executors/openai-api.js +70 -9
  118. package/dist/executors/runtime-fingerprint.js +88 -11
  119. package/dist/executors/script-command.d.ts +8 -0
  120. package/dist/executors/script-command.js +87 -0
  121. package/dist/executors/script.js +202 -29
  122. package/dist/executors/shared.d.ts +35 -3
  123. package/dist/executors/shared.js +113 -15
  124. package/dist/grading/assertions.d.ts +1 -1
  125. package/dist/grading/assertions.js +19 -9
  126. package/dist/grading/diagnostic.d.ts +9 -2
  127. package/dist/grading/diagnostic.js +25 -2
  128. package/dist/grading/index.js +10 -4
  129. package/dist/grading/judge.js +19 -6
  130. package/dist/grading/layered-scores.d.ts +2 -3
  131. package/dist/grading/layered-scores.js +2 -3
  132. package/dist/inputs/load-samples.d.ts +1 -2
  133. package/dist/inputs/load-samples.js +23 -1
  134. package/dist/inputs/mcp-resolver.js +6 -3
  135. package/dist/inputs/sample-document.d.ts +11 -0
  136. package/dist/inputs/sample-document.js +96 -0
  137. package/dist/managed/evidence.d.ts +1 -0
  138. package/dist/managed/evidence.js +1 -1
  139. package/dist/managed/store.js +200 -91
  140. package/dist/observability/codex-trace-adapter.d.ts +5 -0
  141. package/dist/observability/codex-trace-adapter.js +850 -0
  142. package/dist/observability/experience.d.ts +32 -6
  143. package/dist/observability/experience.js +2695 -459
  144. package/dist/observability/feedback-matchers.js +16 -1
  145. package/dist/observability/inbox-view-model.d.ts +1 -1
  146. package/dist/observability/inbox-view-model.js +19 -14
  147. package/dist/observability/inbox.d.ts +7 -1
  148. package/dist/observability/inbox.js +632 -124
  149. package/dist/observability/problem-patterns.js +2 -0
  150. package/dist/observability/review-state.d.ts +6 -0
  151. package/dist/observability/review-state.js +235 -63
  152. package/dist/observability/skill-chain-advisories.js +1 -1
  153. package/dist/observability/skill-chain.js +17 -4
  154. package/dist/observability/skill-health-analyzer.d.ts +32 -7
  155. package/dist/observability/skill-health-analyzer.js +194 -121
  156. package/dist/observability/skill-health-report.d.ts +10 -0
  157. package/dist/observability/skill-health-report.js +620 -0
  158. package/dist/observability/soft-standards/constants.d.ts +0 -1
  159. package/dist/observability/soft-standards/constants.js +0 -1
  160. package/dist/observability/soft-standards/index.d.ts +1 -1
  161. package/dist/observability/soft-standards/index.js +1 -1
  162. package/dist/observability/soft-standards/llm-extractor.js +8 -10
  163. package/dist/observability/soft-standards/skill-standards-store.d.ts +2 -1
  164. package/dist/observability/soft-standards/skill-standards-store.js +59 -18
  165. package/dist/observability/soft-standards/types.d.ts +2 -2
  166. package/dist/observability/trace-adapter.d.ts +12 -7
  167. package/dist/observability/trace-adapter.js +11 -9
  168. package/dist/observability/trace-attribution.d.ts +13 -5
  169. package/dist/observability/trace-attribution.js +315 -21
  170. package/dist/observability/trace-ingestion.d.ts +9 -0
  171. package/dist/observability/trace-ingestion.js +80 -0
  172. package/dist/observability/trace-ir.d.ts +113 -0
  173. package/dist/observability/trace-ir.js +87 -0
  174. package/dist/observability/trace-segmenter.d.ts +19 -6
  175. package/dist/observability/trace-segmenter.js +377 -196
  176. package/dist/observability/trace-session-index.d.ts +19 -0
  177. package/dist/observability/trace-session-index.js +68 -0
  178. package/dist/observability/trace-source.d.ts +12 -4
  179. package/dist/observability/trace-source.js +939 -215
  180. package/dist/renderer/html-renderer.js +37 -6
  181. package/dist/renderer/icons.js +3 -0
  182. package/dist/renderer/observation-inbox-renderer.js +208 -90
  183. package/dist/renderer/skill-detail-renderer.js +452 -109
  184. package/dist/renderer/skill-health-renderer.js +69 -12
  185. package/dist/renderer/summary.js +28 -7
  186. package/dist/renderer/table.js +21 -4
  187. package/dist/renderer/test-view.d.ts +1 -0
  188. package/dist/renderer/test-view.js +44 -9
  189. package/dist/server/indexed-report-store.js +14 -18
  190. package/dist/server/job-store.js +64 -26
  191. package/dist/server/report-server.js +190 -78
  192. package/dist/server/report-store.js +57 -80
  193. package/dist/server/skill-index.js +143 -49
  194. package/dist/server/skill-insights.js +44 -5
  195. package/dist/shared/artifact-graph.d.ts +3 -0
  196. package/dist/shared/artifact-graph.js +224 -0
  197. package/dist/shared/assertion-types.d.ts +8 -0
  198. package/dist/shared/assertion-types.js +46 -0
  199. package/dist/shared/atomic-json.d.ts +8 -0
  200. package/dist/shared/atomic-json.js +33 -0
  201. package/dist/shared/diagnosis-schema.d.ts +9 -0
  202. package/dist/shared/diagnosis-schema.js +181 -0
  203. package/dist/shared/doctor-report.d.ts +3 -0
  204. package/dist/shared/doctor-report.js +103 -0
  205. package/dist/shared/evaluation-job.d.ts +6 -0
  206. package/dist/shared/evaluation-job.js +217 -0
  207. package/dist/shared/executor-result.d.ts +17 -0
  208. package/dist/shared/executor-result.js +221 -0
  209. package/dist/shared/file-lock.d.ts +12 -0
  210. package/dist/shared/file-lock.js +129 -0
  211. package/dist/shared/json-value.d.ts +5 -0
  212. package/dist/shared/json-value.js +36 -0
  213. package/dist/shared/keyed-mutex.d.ts +7 -0
  214. package/dist/shared/keyed-mutex.js +24 -0
  215. package/dist/shared/record-count.d.ts +8 -0
  216. package/dist/shared/record-count.js +43 -0
  217. package/dist/shared/sample-contract.d.ts +3 -0
  218. package/dist/shared/sample-contract.js +332 -0
  219. package/dist/shared/timestamp.d.ts +6 -0
  220. package/dist/shared/timestamp.js +64 -0
  221. package/dist/shared/token-usage.d.ts +19 -0
  222. package/dist/shared/token-usage.js +50 -0
  223. package/dist/shared/tool-call-status.d.ts +8 -0
  224. package/dist/shared/tool-call-status.js +28 -0
  225. package/dist/shared/tool-identity.d.ts +21 -0
  226. package/dist/shared/tool-identity.js +84 -0
  227. package/dist/shared/tool-search.js +73 -16
  228. package/dist/shared/trace-projection.d.ts +5 -0
  229. package/dist/shared/trace-projection.js +20 -0
  230. package/dist/shared/trace-source-kind.d.ts +3 -0
  231. package/dist/shared/trace-source-kind.js +12 -0
  232. package/dist/types/diagnosis.d.ts +2 -0
  233. package/dist/types/eval.d.ts +4 -0
  234. package/dist/types/executor.d.ts +32 -5
  235. package/dist/types/index.d.ts +1 -0
  236. package/dist/types/index.js +1 -0
  237. package/dist/types/judge.d.ts +2 -0
  238. package/dist/types/observability.d.ts +116 -9
  239. package/dist/types/report.d.ts +58 -6
  240. package/dist/types/skill-index.d.ts +7 -0
  241. package/dist/types/trace.d.ts +2 -0
  242. package/dist/types/trace.js +1 -0
  243. package/package.json +9 -5
@@ -27,12 +27,12 @@ export default class Evolve extends BaseCommand {
27
27
  rounds: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
28
28
  target: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
29
29
  samples: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
30
- model: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
31
- 'judge-models': import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
32
- 'improve-model': import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
30
+ model: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
31
+ 'judge-models': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
32
+ 'improve-model': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
33
33
  concurrency: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
34
34
  timeout: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
35
- executor: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
35
+ executor: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
36
36
  'skip-connectivity': import("@oclif/core/interfaces").BooleanFlag<boolean>;
37
37
  effort: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
38
38
  'no-diagnostic': import("@oclif/core/interfaces").BooleanFlag<boolean>;
@@ -7,6 +7,8 @@ import { enumStringParser, integerStringParser, numberStringParser } from '../oc
7
7
  import { CliExit } from '../lib/cli-exit.js';
8
8
  import { tCli } from '../lib/i18n.js';
9
9
  import { makeOnProgress } from '../lib/progress.js';
10
+ import { formatSampleGenerationFailureHint } from '../lib/generation-failure-hint.js';
11
+ import { envJudgeModels, resolveRuntimeSelection } from '../lib/runtime-defaults.js';
10
12
  /** 受管联动旁路:evolve 写回 source 后记证据 + re-baseline。任何异常都不该让 evolve 失败,
11
13
  * 故 try/catch 吞掉、返回 null(同 eval 的 recordEvidenceSafely 口径)。 */
12
14
  async function recordEvolveOutcomeSafely(input) {
@@ -163,7 +165,10 @@ export async function runEvolve(args, flags, lang) {
163
165
  catch (err) {
164
166
  if (err instanceof CliExit)
165
167
  throw err;
166
- console.error(tCli('cli.common.error_prefix', lang, { message: err.message }));
168
+ const message = err.message;
169
+ console.error(tCli('cli.common.error_prefix', lang, {
170
+ message: `${message}${formatSampleGenerationFailureHint(message, flags.executor, lang)}`,
171
+ }));
167
172
  throw new CliExit(1);
168
173
  }
169
174
  }
@@ -346,24 +351,21 @@ export default class Evolve extends BaseCommand {
346
351
  }),
347
352
  model: Flags.string({
348
353
  description: bilingual({
349
- zh: '被评测的 LLM,默认 sonnet。无用例时也用作自动生成用例的出题模型。',
350
- en: 'Evaluated LLM, default sonnet. Also used as the sample-generation model when no samples exist.',
354
+ zh: '被评测的 LLM。Codex 自动读取本机配置;无用例时也用作自动生成用例的出题模型。',
355
+ en: 'Evaluated LLM. Codex reads the local configured model. Also used to generate samples when none exist.',
351
356
  }),
352
- default: 'sonnet',
353
357
  }),
354
358
  'judge-models': Flags.string({
355
359
  description: bilingual({
356
- zh: '评委 model(单评委约束),格式 executor:model。默认 claude:haiku',
357
- en: 'Judge model (single judge required), executor:model format. Default claude:haiku',
360
+ zh: '评委 model(单评委约束),格式 executor:model。默认跟随所选执行器;Codex 沿用被测模型。',
361
+ en: 'Judge model (single judge required), executor:model format. Defaults to the selected executor; Codex reuses the evaluated model.',
358
362
  }),
359
- default: 'claude:haiku',
360
363
  }),
361
364
  'improve-model': Flags.string({
362
365
  description: bilingual({
363
- zh: '负责重写 skill 的 LLM,默认 sonnet',
364
- en: 'LLM that rewrites the skill, default sonnet',
366
+ zh: '负责重写 skill 的 LLM,默认沿用被测模型',
367
+ en: 'LLM that rewrites the skill; defaults to the evaluated model',
365
368
  }),
366
- default: 'sonnet',
367
369
  }),
368
370
  concurrency: Flags.string({
369
371
  description: bilingual({ zh: '评测并发数,默认 1', en: 'Eval concurrency, default 1' }),
@@ -376,8 +378,10 @@ export default class Evolve extends BaseCommand {
376
378
  parse: numberStringParser('--timeout', { min: 1 }),
377
379
  }),
378
380
  executor: Flags.string({
379
- description: bilingual({ zh: '执行器名,默认 claude', en: 'Executor name, default claude' }),
380
- default: 'claude',
381
+ description: bilingual({
382
+ zh: '执行器名。Codex 任务内自动用 codex;也可用 OMK_EXECUTOR 设置环境偏好。',
383
+ en: 'Executor name. Defaults to codex inside Codex tasks; OMK_EXECUTOR sets an environment preference.',
384
+ }),
381
385
  }),
382
386
  'skip-connectivity': Flags.boolean({
383
387
  description: bilingual({
@@ -509,7 +513,17 @@ export default class Evolve extends BaseCommand {
509
513
  const { args, flags } = await this.parse(Evolve);
510
514
  const lang = this.lang;
511
515
  await this.runWithCliExit(async () => {
512
- await runEvolve(args, { ...flags, lang }, lang);
516
+ const runtime = resolveRuntimeSelection({ executor: flags.executor, model: flags.model }, { lang });
517
+ await runEvolve(args, {
518
+ ...flags,
519
+ executor: runtime.executor,
520
+ model: runtime.model,
521
+ 'judge-models': flags['judge-models']
522
+ ?? envJudgeModels()
523
+ ?? `${runtime.executor}:${runtime.judgeModel}`,
524
+ 'improve-model': flags['improve-model'] ?? runtime.model,
525
+ lang,
526
+ }, lang);
513
527
  });
514
528
  }
515
529
  }
@@ -1,8 +1,9 @@
1
- import { resolve, join } from 'node:path';
1
+ import { resolve, join, relative, sep } from 'node:path';
2
2
  import { Args } from '@oclif/core';
3
3
  import { LANG_FLAG, bilingual, resolveLang } from '../oclif/i18n.js';
4
4
  import { BaseCommand } from '../oclif/base-command.js';
5
5
  import { tCli } from '../lib/i18n.js';
6
+ import { shellQuoteArg } from '../../shared/shell-quote.js';
6
7
  // 预置 .omk/.gitignore:测量 bulk + doctor --fix 备份(项目本地、不该入库)默认不入库;
7
8
  // managed/ 治理档案 + 配置不在此列,默认 track。
8
9
  const INIT_OMK_GITIGNORE = `# omk 测量 bulk + doctor --fix 备份(项目本地)——不入库;前导 / 锚定 .omk/ 顶层,不误伤嵌套同名目录。
@@ -94,6 +95,17 @@ description: 多维度代码审查,覆盖安全 / 健壮 / 可维护 / 性能,
94
95
 
95
96
  对每个维度给出具体的改进建议,并标注严重程度(高/中/低)。
96
97
  `;
98
+ const INIT_EVAL_COMMAND = 'omk eval --control code-review-v1 --treatment code-review-v2';
99
+ function isOutsideCwd(relPath) {
100
+ return relPath === '..' || relPath.startsWith(`..${sep}`);
101
+ }
102
+ function nextEvalCommand(targetDir) {
103
+ const rel = relative(resolve('.'), targetDir);
104
+ if (!rel)
105
+ return INIT_EVAL_COMMAND;
106
+ const cdTarget = isOutsideCwd(rel) ? targetDir : rel;
107
+ return `cd ${shellQuoteArg(cdTarget)} && ${INIT_EVAL_COMMAND}`;
108
+ }
97
109
  export default class Init extends BaseCommand {
98
110
  static description = bilingual({
99
111
  zh: '初始化一个 omk 项目:在目标目录铺好待测知识载体(skills/)与评测用例(eval-samples.json),供 omk eval / doctor / evolve / observe / list 操作。默认是两版 code-review skill 的 A/B 起步模板。',
@@ -158,10 +170,11 @@ export default class Init extends BaseCommand {
158
170
  console.log(tCli('cli.init.scaffolded', lang, { dir: targetDir }));
159
171
  console.log('');
160
172
  console.log(tCli('cli.init.next_steps_title', lang));
161
- console.log(tCli('cli.init.next_step_run', lang));
173
+ console.log(tCli('cli.init.next_step_run', lang, { command: nextEvalCommand(targetDir) }));
162
174
  console.log(tCli('cli.init.next_step_executor', lang));
175
+ console.log(tCli('cli.init.next_step_report', lang));
163
176
  console.log(tCli('cli.init.next_step_customize', lang));
164
- console.log(tCli('cli.init.note_codex_executor', lang));
177
+ console.log(tCli('cli.init.note_skill_injection', lang));
165
178
  });
166
179
  }
167
180
  }
@@ -3,21 +3,20 @@ import { Flags } from '@oclif/core';
3
3
  import { BaseCommand } from '../../oclif/base-command.js';
4
4
  import { LANG_FLAG, bilingual } from '../../oclif/i18n.js';
5
5
  import { integerStringParser } from '../../oclif/parsers.js';
6
+ import { resolveRuntimeSelection } from '../../lib/runtime-defaults.js';
6
7
  import { shellQuoteArg } from '../../../shared/shell-quote.js';
8
+ import { ownRecordValue } from '../../../shared/record-count.js';
7
9
  function pickSkillCount(value, skillName) {
8
- if (!value || value[skillName] == null)
9
- return undefined;
10
- return { [skillName]: value[skillName] };
10
+ const selected = value ? ownRecordValue(value, skillName) : undefined;
11
+ return selected == null ? undefined : { [skillName]: selected };
11
12
  }
12
13
  function pickSkillString(value, skillName) {
13
- if (!value || value[skillName] == null)
14
- return undefined;
15
- return { [skillName]: value[skillName] };
14
+ const selected = value ? ownRecordValue(value, skillName) : undefined;
15
+ return selected == null ? undefined : { [skillName]: selected };
16
16
  }
17
17
  function pickSkillToolCounts(value, skillName) {
18
- if (!value || value[skillName] == null)
19
- return undefined;
20
- return { [skillName]: value[skillName] };
18
+ const selected = value ? ownRecordValue(value, skillName) : undefined;
19
+ return selected == null ? undefined : { [skillName]: selected };
21
20
  }
22
21
  // runObserveInbox export:test/cli/observe.test.ts in-process import 验证 by-skill 聚合行为。
23
22
  export async function runObserveInbox(_args, flags, lang) {
@@ -28,19 +27,29 @@ export async function runObserveInbox(_args, flags, lang) {
28
27
  : (flags.global ? DEFAULT_GLOBAL_OBSERVATIONS_DIR : DEFAULT_OBSERVATIONS_DIR);
29
28
  if (flags['llm-enhanced-review']) {
30
29
  const { buildObservationInboxViewModel } = await import('../../../observability/inbox-view-model.js');
31
- const { extractSkillSoftStandards, DEFAULT_LLM_ENHANCED_REVIEW_MODEL } = await import('../../../observability/soft-standards/index.js');
30
+ const { extractSkillSoftStandards } = await import('../../../observability/soft-standards/index.js');
32
31
  const view = buildObservationInboxViewModel(dir, { skill: flags.skill });
33
32
  const candidates = Object.values(view.skillChains)
34
33
  .map((chain) => ({ chain, runtimeEvidence: buildLlmEnhancedRuntimeEvidence(view, chain.skillName) }))
35
34
  .filter(({ runtimeEvidence }) => hasLlmEnhancedRuntimeEvidence(runtimeEvidence));
35
+ if (candidates.length === 0) {
36
+ if (flags.json) {
37
+ console.log(JSON.stringify({ kind: 'observe-llm-enhanced-review', records: [] }, null, 2));
38
+ }
39
+ else {
40
+ console.log(lang === 'zh' ? '没有可用于 LLM 增强复盘的运行证据' : 'No runtime evidence is available for LLM enhanced review');
41
+ }
42
+ return;
43
+ }
44
+ const runtime = resolveRuntimeSelection({ executor: flags.executor, model: flags.model }, { lang });
36
45
  const records = [];
37
46
  for (const { chain, runtimeEvidence } of candidates) {
38
47
  records.push(await extractSkillSoftStandards({
39
48
  observationsDir: dir,
40
49
  skillChain: chain,
41
50
  runtimeEvidence,
42
- model: flags.model || DEFAULT_LLM_ENHANCED_REVIEW_MODEL,
43
- executorName: flags.executor,
51
+ model: runtime.model,
52
+ executorName: runtime.executor,
44
53
  refresh: flags.refresh,
45
54
  }));
46
55
  }
@@ -48,10 +57,6 @@ export async function runObserveInbox(_args, flags, lang) {
48
57
  console.log(JSON.stringify({ kind: 'observe-llm-enhanced-review', records }, null, 2));
49
58
  return;
50
59
  }
51
- if (records.length === 0) {
52
- console.log(lang === 'zh' ? '没有可用于 LLM 增强复盘的运行证据' : 'No runtime evidence is available for LLM enhanced review');
53
- return;
54
- }
55
60
  console.log(lang === 'zh' ? 'LLM 增强复盘已生成:' : 'LLM enhanced review generated:');
56
61
  for (const record of records) {
57
62
  console.log(`- ${record.skillName} standards=${record.standards.length} model=${record.model} prompt=${record.promptId}/${record.promptVersion}`);
@@ -120,8 +125,10 @@ export async function runObserveInbox(_args, flags, lang) {
120
125
  for (const item of items) {
121
126
  const evidence = item.evidence.query || item.evidence.path || item.evidence.assistantSnippet || item.evidence.outputSnippet || '';
122
127
  const artifactVersion = item.artifactVersion === 'unknown' ? '⚠ unknown' : item.artifactVersion;
128
+ const timestampedOccurrences = item.timestampedOccurrences
129
+ ?? (item.firstSeen === '1970-01-01T00:00:00.000Z' ? 0 : item.occurrences);
123
130
  console.log(`- [${item.severity}] (${item.sourceKind}) ${item.skillName} ${item.signalType}/${item.signalSubtype} x${item.occurrences} confidence=${item.confidence.toFixed(2)} attribution=${item.attributionConfidence.toFixed(2)}`);
124
- console.log(` lastSeen=${item.lastSeen} version=${artifactVersion}`);
131
+ console.log(` lastSeen=${timestampedOccurrences > 0 ? item.lastSeen : 'unknown'} version=${artifactVersion}`);
125
132
  console.log(` reason=${item.severityReasonCode ?? 'unknown'}`);
126
133
  if (evidence)
127
134
  console.log(` evidence=${evidence.slice(0, 180)}`);
@@ -260,14 +267,14 @@ export default class ObserveInbox extends BaseCommand {
260
267
  }),
261
268
  model: Flags.string({
262
269
  description: bilingual({
263
- zh: 'LLM 增强复盘使用的模型,默认 sonnet',
264
- en: 'Model for LLM enhanced review, default sonnet',
270
+ zh: 'LLM 增强复盘使用的模型。Codex 自动读取本机配置;也可用 OMK_MODEL 设置环境偏好。',
271
+ en: 'Model for LLM enhanced review. Codex reads the local configured model; OMK_MODEL sets an environment preference.',
265
272
  }),
266
273
  }),
267
274
  executor: Flags.string({
268
275
  description: bilingual({
269
- zh: 'LLM 增强复盘使用的执行器',
270
- en: 'Executor for LLM enhanced review',
276
+ zh: 'LLM 增强复盘使用的执行器。Codex 任务内自动用 codex;也可用 OMK_EXECUTOR 设置环境偏好。',
277
+ en: 'Executor for LLM enhanced review. Defaults to codex inside Codex tasks; OMK_EXECUTOR sets an environment preference.',
271
278
  }),
272
279
  }),
273
280
  json: Flags.boolean({
@@ -1,4 +1,4 @@
1
- import { mkdirSync, writeFileSync } from 'node:fs';
1
+ import { mkdirSync } from 'node:fs';
2
2
  import { resolve } from 'node:path';
3
3
  import { Args, Flags } from '@oclif/core';
4
4
  import { LANG_FLAG, bilingual } from '../../oclif/i18n.js';
@@ -10,6 +10,7 @@ import { projectObserveHealthDir, globalObserveHealthDir } from '../../../eval-c
10
10
  import { indexObserveWrite } from '../../../eval-core/artifact-index.js';
11
11
  import { reportFilePath, runFileSuffix } from '../../../eval-core/artifact-file-names.js';
12
12
  import { migrateLegacyReportFiles } from '../../../eval-core/report-file-migration.js';
13
+ import { writeJsonFileAtomic } from '../../../shared/atomic-json.js';
13
14
  /**
14
15
  * observe-health 报告落盘:id / 文件名加 4 位随机段,根治「同秒两次 omk observe 直接覆盖、数据丢失」的 bug。
15
16
  * 文件名统一为 `{run}.report.json`:目录表达 observe-health 域,文件表达这是可读报告。
@@ -20,7 +21,7 @@ export function persistObserveHealthReport(report, outDir) {
20
21
  migrateLegacyReportFiles(outDir, 'observe-health');
21
22
  const id = runFileSuffix();
22
23
  const jsonPath = reportFilePath(outDir, id);
23
- writeFileSync(jsonPath, JSON.stringify(report, null, 2));
24
+ writeJsonFileAtomic(jsonPath, report);
24
25
  indexObserveWrite(report, jsonPath, outDir, id);
25
26
  return { id, jsonPath };
26
27
  }
@@ -90,23 +91,23 @@ async function recordObserveFeedback(report, reportId, lang) {
90
91
  // 由 Studio 健康报告页按需渲染。observe 这条线的另一条产物是观测收件箱(observe-inbox),走子命令 ingest / inbox / show。
91
92
  export default class Observe extends BaseCommand {
92
93
  static description = bilingual({
93
- zh: '分析 sessions 目录的 skill 调用健康度(默认行为)。子命令:ingest / inbox / show。',
94
- en: 'Analyze skill invocation health from a sessions dir (default). Subcommands: ingest / inbox / show.',
94
+ zh: '把 Codex、Claude Code、OpenClaw 或 markdown trace 统一为 Trace IR,分析 skill 调用健康度(默认行为)。子命令:ingest / inbox / show。',
95
+ en: 'Normalize Codex, Claude Code, OpenClaw, or markdown traces into Trace IR and analyze skill invocation health (default). Subcommands: ingest / inbox / show.',
95
96
  });
96
97
  static examples = [
97
98
  {
98
99
  description: bilingual({
99
- zh: '分析最近 7 天',
100
- en: 'Analyze last 7 days',
100
+ zh: '分析最近 7 天的 Codex rollout',
101
+ en: 'Analyze Codex rollouts from the last 7 days',
101
102
  }),
102
- command: '<%= config.bin %> observe ~/.claude/sessions --last 7d',
103
+ command: '<%= config.bin %> observe ~/.codex/sessions --last 7d',
103
104
  },
104
105
  ];
105
106
  static args = {
106
107
  sessionsDir: Args.string({
107
108
  description: bilingual({
108
- zh: 'sessions 目录路径(如 ~/.claude/sessions)',
109
- en: 'Sessions dir path (e.g. ~/.claude/sessions)',
109
+ zh: 'sessions 目录路径(如 ~/.codex/sessions 或 ~/.claude/projects/<project>)',
110
+ en: 'Sessions dir path (e.g. ~/.codex/sessions or ~/.claude/projects/<project>)',
110
111
  }),
111
112
  required: false,
112
113
  }),
@@ -192,9 +193,17 @@ export default class Observe extends BaseCommand {
192
193
  ? resolve(flags['output-dir'])
193
194
  : (flags.global ? globalObserveHealthDir() : projectObserveHealthDir());
194
195
  const { id, jsonPath } = persistObserveHealthReport(report, outDir);
195
- const { sessionCount, segmentCount, toolCallCount, toolFailureRate } = report.meta;
196
+ const { traceIngestionNotices } = await import('../../../observability/trace-ingestion.js');
197
+ for (const notice of traceIngestionNotices(report.meta.ingestion, lang)) {
198
+ process.stderr.write(`${notice.text}\n`);
199
+ }
200
+ const { sessionCount, segmentCount, toolCallCount, toolFailureRate, toolResolvedCount = toolCallCount, toolCancelledCount = 0, toolUnknownCount = 0, } = report.meta;
201
+ const toolComparableCount = Math.max(0, toolResolvedCount - toolCancelledCount);
202
+ const failureSummary = toolCallCount > 0 && toolComparableCount === 0
203
+ ? `fail rate: unavailable${toolCancelledCount > 0 ? ` · cancelled: ${toolCancelledCount}` : ''}${toolUnknownCount > 0 ? ` · unknown outcomes: ${toolUnknownCount}` : ''}`
204
+ : `fail rate: ${(toolFailureRate * 100).toFixed(1)}% (${toolComparableCount} comparable${toolCancelledCount > 0 ? ` · ${toolCancelledCount} cancelled` : ''})`;
196
205
  console.log('');
197
- console.log(`sessions: ${sessionCount} · segments: ${segmentCount} · tool calls: ${toolCallCount} · fail rate: ${(toolFailureRate * 100).toFixed(1)}%`);
206
+ console.log(`sessions: ${sessionCount} · segments: ${segmentCount} · tool calls: ${toolCallCount} · ${failureSummary}`);
198
207
  const overallConf = report.overall.confidence;
199
208
  const confSuffix = overallConf === 'high'
200
209
  ? ''
@@ -1,4 +1,6 @@
1
1
  import { BaseCommand } from '../../oclif/base-command.js';
2
+ import type { ObservationInboxReport } from '../../../types/observability.js';
3
+ export declare function formatIngestSummary(report: ObservationInboxReport, lang: 'zh' | 'en'): string;
2
4
  export default class ObserveIngest extends BaseCommand {
3
5
  static description: string;
4
6
  static args: {
@@ -8,6 +10,7 @@ export default class ObserveIngest extends BaseCommand {
8
10
  lang: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
9
11
  'output-dir': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
10
12
  global: import("@oclif/core/interfaces").BooleanFlag<boolean>;
13
+ json: import("@oclif/core/interfaces").BooleanFlag<boolean>;
11
14
  };
12
15
  run(): Promise<void>;
13
16
  }
@@ -4,6 +4,21 @@ import { BaseCommand } from '../../oclif/base-command.js';
4
4
  import { LANG_FLAG, bilingual } from '../../oclif/i18n.js';
5
5
  import { CliExit } from '../../lib/cli-exit.js';
6
6
  import { shellQuoteArg } from '../../../shared/shell-quote.js';
7
+ export function formatIngestSummary(report, lang) {
8
+ const severityCounts = {
9
+ high: 0,
10
+ medium: 0,
11
+ low: 0,
12
+ noise: 0,
13
+ };
14
+ for (const item of report.items)
15
+ severityCounts[item.severity]++;
16
+ const sessions = report.meta.sessionCount ?? 0;
17
+ if (lang === 'zh') {
18
+ return `observe inbox:会话 ${sessions} · 片段 ${report.meta.segmentCount} · 信号 ${report.meta.itemCount}(高 ${severityCounts.high} / 中 ${severityCounts.medium} / 低 ${severityCounts.low} / 噪声 ${severityCounts.noise})`;
19
+ }
20
+ return `observe inbox: ${sessions} sessions · ${report.meta.segmentCount} segments · ${report.meta.itemCount} signals (high ${severityCounts.high} / medium ${severityCounts.medium} / low ${severityCounts.low} / noise ${severityCounts.noise})`;
21
+ }
7
22
  export default class ObserveIngest extends BaseCommand {
8
23
  static description = bilingual({
9
24
  zh: '把 trace 目录 ingest 成 observation inbox 报告。',
@@ -33,6 +48,13 @@ export default class ObserveIngest extends BaseCommand {
33
48
  }),
34
49
  default: false,
35
50
  }),
51
+ json: Flags.boolean({
52
+ description: bilingual({
53
+ zh: '把完整 observation inbox 报告输出到 stdout;默认只输出摘要。',
54
+ en: 'Print the full observation inbox report to stdout; defaults to a concise summary.',
55
+ }),
56
+ default: false,
57
+ }),
36
58
  };
37
59
  async run() {
38
60
  const { args, flags } = await this.parse(ObserveIngest);
@@ -62,7 +84,7 @@ export default class ObserveIngest extends BaseCommand {
62
84
  : 'Error: --output-dir must not be an empty string.');
63
85
  throw new CliExit(2);
64
86
  }
65
- const { buildObservationInboxReport, saveObservationInboxReport, DEFAULT_PROJECT_OBSERVATIONS_DIR, DEFAULT_GLOBAL_OBSERVATIONS_DIR } = await import('../../../observability/inbox.js');
87
+ const { buildObservationInboxReport, compactObservationInboxReport, saveObservationInboxReport, DEFAULT_PROJECT_OBSERVATIONS_DIR, DEFAULT_GLOBAL_OBSERVATIONS_DIR, } = await import('../../../observability/inbox.js');
66
88
  // 显式 --output-dir 最高;否则 --global 写全局、默认写项目(读侧 loadObservationInboxReports 会从项目兜底到全局)。
67
89
  const defaultDir = flags.global ? DEFAULT_GLOBAL_OBSERVATIONS_DIR : DEFAULT_PROJECT_OBSERVATIONS_DIR;
68
90
  const outDir = resolve(outDirRaw ?? defaultDir);
@@ -71,7 +93,13 @@ export default class ObserveIngest extends BaseCommand {
71
93
  const report = buildObservationInboxReport(tracePath, { reviewState: loadObservationReviewState(outDir) });
72
94
  report.diagnostics = buildObserveDiagnosticsFromReport(report);
73
95
  const path = saveObservationInboxReport(report, outDir);
74
- console.log(JSON.stringify(report, null, 2));
96
+ console.log(flags.json
97
+ ? JSON.stringify(compactObservationInboxReport(report), null, 2)
98
+ : formatIngestSummary(report, lang));
99
+ const { traceIngestionNotices } = await import('../../../observability/trace-ingestion.js');
100
+ for (const notice of traceIngestionNotices(report.meta.ingestion, lang)) {
101
+ process.stderr.write(`${notice.text}\n`);
102
+ }
75
103
  const inboxCommand = `omk observe inbox --input-dir ${shellQuoteArg(outDir)}`;
76
104
  const sampleCommand = `omk sample --from-traces --observations-dir ${shellQuoteArg(outDir)}`;
77
105
  process.stderr.write(lang === 'zh'
@@ -1,8 +1,10 @@
1
1
  import { BaseCommand } from '../oclif/base-command.js';
2
2
  import { type CliLang } from '../lib/i18n.js';
3
- import { type LoadSamplesResult } from '../../inputs/load-samples.js';
3
+ import { writeFixedSamplesToSources } from '../../inputs/sample-document.js';
4
4
  import type { SampleFlags } from '../lib/cmd-flags.js';
5
5
  import type { Report, Sample as SampleType } from '../../types/index.js';
6
+ import type { ResolvedSkillInput } from '../lib/resolve-skill-input.js';
7
+ export declare function sampleNextEvalCommand(resolved: Pick<ResolvedSkillInput, 'isDirectorySkill' | 'skillDir' | 'skillPath'>): string;
6
8
  /** --append 合并:已有用例原样保留,新用例逐条接在后面;sample_id 撞已有(或本批已用)时
7
9
  * 自动加 `-2`/`-3` 后缀去重。模型每次从 s001 重编号,撞 id 不代表内容重复,所以是改名保留
8
10
  * 而非丢弃(不做内容级去重)。`reserved` 为额外要避开的 id 集(目录模式跨同目录其它 sample
@@ -21,10 +23,11 @@ export declare function assertFixReportMatchesCurrentInputs(params: {
21
23
  treatmentName: string;
22
24
  currentContentHash: string;
23
25
  samples: SampleType[];
26
+ samplesBaseDir?: string;
24
27
  sampleIds: Set<string>;
25
28
  lang?: CliLang;
26
29
  }): void;
27
- export declare function writeFixedSamplesToSources(loaded: Pick<LoadSamplesResult, 'sourceFiles' | 'sampleSourceById'>, samples: SampleType[], changedIds: Set<string>): string[];
30
+ export { writeFixedSamplesToSources };
28
31
  export declare function runSampleFromTraces(flags: SampleFlags, lang: CliLang): Promise<void>;
29
32
  export default class Sample extends BaseCommand {
30
33
  static description: string;
@@ -39,7 +42,7 @@ export default class Sample extends BaseCommand {
39
42
  lang: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
40
43
  batch: import("@oclif/core/interfaces").BooleanFlag<boolean>;
41
44
  count: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
42
- model: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
45
+ model: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
43
46
  executor: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
44
47
  'skill-dir': import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
45
48
  focus: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;