oh-my-knowledge 0.44.0 → 0.46.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/README.md +1 -1
  2. package/README.zh.md +1 -1
  3. package/dist/artifact-graph/doctor.js +1 -1
  4. package/dist/artifact-graph/eval.js +120 -0
  5. package/dist/assets/agent-skills/omk/SKILL.md +3 -3
  6. package/dist/assets/agent-skills/omk/references/commands.md +13 -6
  7. package/dist/cli/commands/doctor.d.ts +3 -2
  8. package/dist/cli/commands/doctor.js +66 -63
  9. package/dist/cli/lib/cmd-flags.d.ts +3 -2
  10. package/dist/cli/lib/i18n-dict/common.d.ts +1 -1
  11. package/dist/cli/lib/i18n-dict/common.js +0 -4
  12. package/dist/doctor/endpoint-rule.js +1 -1
  13. package/dist/doctor/health/composer.js +162 -50
  14. package/dist/doctor/health/consensus.d.ts +43 -0
  15. package/dist/doctor/health/consensus.js +224 -0
  16. package/dist/doctor/health/dimension-spec.d.ts +6 -0
  17. package/dist/doctor/health/register.d.ts +0 -1
  18. package/dist/doctor/health/register.js +0 -1
  19. package/dist/doctor/index.js +26 -3
  20. package/dist/doctor/messages.d.ts +1 -1
  21. package/dist/doctor/messages.js +8 -0
  22. package/dist/doctor/rules.d.ts +4 -2
  23. package/dist/doctor/rules.js +4 -2
  24. package/dist/eval-core/evaluation-reporting.js +1 -0
  25. package/dist/executors/claude-cli.js +1 -1
  26. package/dist/inputs/load-samples.js +27 -0
  27. package/dist/observability/skill-chain-advisories.js +2 -2
  28. package/dist/renderer/doctor-detail-renderer.js +30 -2
  29. package/dist/renderer/report-shell.d.ts +1 -1
  30. package/dist/renderer/report-shell.js +5 -0
  31. package/dist/renderer/skill-detail-renderer.js +908 -94
  32. package/dist/renderer/skill-list-renderer.js +1 -1
  33. package/dist/server/skill-index.js +58 -1
  34. package/dist/shared/llm-prompts/registry.js +2 -1
  35. package/dist/shared/llm-prompts/skill-health-merge.d.ts +22 -0
  36. package/dist/shared/llm-prompts/skill-health-merge.js +79 -0
  37. package/dist/shared/llm-prompts/skill-health.js +1 -1
  38. package/dist/types/artifact-graph.d.ts +1 -1
  39. package/dist/types/doctor.d.ts +32 -8
  40. package/dist/types/eval.d.ts +10 -0
  41. package/dist/types/report.d.ts +3 -1
  42. package/dist/types/skill-index.d.ts +14 -0
  43. package/package.json +6 -5
package/README.md CHANGED
@@ -102,7 +102,7 @@ RAG-specific evals: see RAGAS (separate niche, complementary to omk). Full compa
102
102
  | **30+ assertion types** | substring, regex, JSON Schema, ROUGE/BLEU/Levenshtein similarity, agent tool-call assertions, semantic similarity, custom JS |
103
103
  | **Statistical rigor** | Bootstrap CI / length-debias / saturation curve on by default; Krippendorff α auto-computed with a gold set. [Details →](docs/explanation/statistical-rigor.md) |
104
104
  | **RAG metrics** | `faithfulness` / `answer_relevancy` / `context_recall` — anti-hallucination + answer relevance + context coverage |
105
- | **LLM health audit** | `omk doctor` grades 7 builtin dimensions; `--static-only` runs offline without an LLM |
105
+ | **LLM health audit** | `omk doctor` grades 7 builtin dimensions; repeats the audit (`--repeat`) and merges findings by k/n consensus |
106
106
  | **Production observability** | parse Claude Code session JSONL traces; measure per-skill failure rate / latency / cost / knowledge-gap signals |
107
107
  | **Knowledge-gap detection** | severity-weighted signals quantify risk exposure instead of claiming completeness |
108
108
  | **Construct-validity isolation** | `--strict-baseline` (default ON) cuts three contamination channels so baseline doesn't silently see the skill it's being compared against |
package/README.zh.md CHANGED
@@ -102,7 +102,7 @@ RAG 专项评测请看 RAGAS(独立 niche,跟 omk 互补)。完整对比
102
102
  | **30+ 种断言** | 包含子串、正则、JSON Schema、ROUGE/BLEU/Levenshtein 相似度、Agent 工具调用、语义相似度、自定义函数等 |
103
103
  | **统计严谨性** | Bootstrap CI / 长度去偏 / 饱和曲线默认开,Krippendorff α 提供 gold 集即自动计算。[详情 →](docs/zh/explanation/statistical-rigor.md) |
104
104
  | **RAG metrics** | `faithfulness` / `answer_relevancy` / `context_recall` 三 metric — 反幻觉 + 切题度 + context 覆盖 |
105
- | **LLM 健康度审计** | `omk doctor` 给 7 个内置维度独立打分;`--static-only` 可离线无 LLM 调用 |
105
+ | **LLM 健康度审计** | `omk doctor` 给 7 个内置维度独立打分;重复采样(`--repeat`)+ k/n 共识归并 |
106
106
  | **线上 session 观测** | 解析 Claude Code session JSONL,测量各 skill 的失败率、耗时、token 成本、知识缺口信号 |
107
107
  | **知识缺口识别** | 严重度加权的信号量化风险敞口,不宣称完备性 |
108
108
  | **用例隔离 (construct validity)** | `--strict-baseline`(默认开)三堵 baseline 拿到被测 skill 的污染路径 |
@@ -506,7 +506,7 @@ export function renderDoctorEvidenceCard(graph, skill, lang) {
506
506
  ];
507
507
  const hiddenStructure = renderStructureDetails(graph, lang);
508
508
  return [
509
- `## ${zh ? 'Skill Evidence Card' : 'Skill Evidence Card'}:${skill.skillName}`,
509
+ `## ${zh ? '知识图谱摘要' : 'Skill Map Summary'}${zh ? ':' : ': '}${skill.skillName}`,
510
510
  '',
511
511
  statusSentence,
512
512
  '',
@@ -133,8 +133,73 @@ function sampleAttrs(snapshot) {
133
133
  display.tripwire = true;
134
134
  if (snapshot.assertions?.length)
135
135
  display.assertionCount = snapshot.assertions.length;
136
+ if (snapshot.covers?.length)
137
+ display.declaredCoverageTargetCount = snapshot.covers.length;
136
138
  return Object.keys(display).length > 0 ? { display } : undefined;
137
139
  }
140
+ const COVERAGE_TARGET_NODE_KIND = {
141
+ skill: 'skill',
142
+ skill_file: 'skill_file',
143
+ frontmatter: 'frontmatter',
144
+ reference: 'reference',
145
+ script: 'script',
146
+ hard_rule: 'hard_rule',
147
+ workflow: 'workflow',
148
+ workflow_node: 'workflow_node',
149
+ };
150
+ function normalizeCoverageRef(target) {
151
+ const raw = target.ref.trim().replaceAll('\\', '/');
152
+ if (target.targetKind === 'reference' || target.targetKind === 'script' || target.targetKind === 'skill_file') {
153
+ return raw.replace(/^\/+/, '').replace(/^\.\//, '');
154
+ }
155
+ return raw;
156
+ }
157
+ function coverageTargetStableKey(target, artifactHash) {
158
+ const ref = normalizeCoverageRef(target);
159
+ switch (target.targetKind) {
160
+ case 'skill':
161
+ return `v1:skill:${artifactHash}`;
162
+ case 'skill_file':
163
+ return `v1:skill-file:${artifactHash}:${ref || 'SKILL.md'}`;
164
+ case 'frontmatter':
165
+ return `v1:frontmatter:${artifactHash}`;
166
+ case 'reference':
167
+ return `v1:reference:${artifactHash}:${ref}`;
168
+ case 'script':
169
+ return `v1:script:${artifactHash}:${ref}`;
170
+ case 'hard_rule':
171
+ return `v1:hard-rule:${artifactHash}:${ref}`;
172
+ case 'workflow':
173
+ return `v1:workflow:${artifactHash}:${ref}`;
174
+ case 'workflow_node':
175
+ return `v1:workflow-node:${artifactHash}:${ref}`;
176
+ }
177
+ }
178
+ function coverageTargetLabel(target) {
179
+ const ref = normalizeCoverageRef(target);
180
+ switch (target.targetKind) {
181
+ case 'skill':
182
+ return ref && ref !== 'skill' ? ref : 'SKILL.md';
183
+ case 'skill_file':
184
+ return ref || 'SKILL.md';
185
+ case 'frontmatter':
186
+ return 'frontmatter';
187
+ default:
188
+ return ref;
189
+ }
190
+ }
191
+ function coverageEvidence(report, sampleId, targetIndex, target) {
192
+ return [{
193
+ sourceKind: 'sample',
194
+ sourceId: sampleId,
195
+ selector: {
196
+ selectorKind: 'json-pointer',
197
+ value: `/sampleSnapshots/${jsonPointerToken(sampleId)}/covers/${targetIndex}`,
198
+ },
199
+ contentHash: report.meta.sampleHashes?.[sampleId],
200
+ label: `${sampleId} covers ${target.targetKind}:${normalizeCoverageRef(target)}`,
201
+ }];
202
+ }
138
203
  export function evalGraphDirForReportOutput(reportOutputDir) {
139
204
  return basename(reportOutputDir) === 'reports'
140
205
  ? join(dirname(reportOutputDir), 'graphs', 'eval')
@@ -147,6 +212,16 @@ export function buildEvalArtifactGraph(options) {
147
212
  const edges = [];
148
213
  const nodeIdsByStableKey = new Map();
149
214
  const configs = variantConfigByName(report);
215
+ const skillArtifacts = report.meta.variants
216
+ .map((variant) => ({
217
+ variant,
218
+ artifactHash: report.meta.artifactHashes?.[variant],
219
+ config: configs.get(variant),
220
+ }))
221
+ .filter((item) => item.config?.artifactKind === 'skill'
222
+ && typeof item.artifactHash === 'string'
223
+ && item.artifactHash.length > 0
224
+ && item.artifactHash !== 'no-skill');
150
225
  const addNode = (stableKey, nodeKind, nodeRole, label, extra = {}) => {
151
226
  const existing = nodeIdsByStableKey.get(stableKey);
152
227
  if (existing)
@@ -175,6 +250,50 @@ export function buildEvalArtifactGraph(options) {
175
250
  ...extra,
176
251
  });
177
252
  };
253
+ const addCoverageEdges = (sampleId, sampleNodeId, snapshot) => {
254
+ if (!snapshot.covers?.length || skillArtifacts.length === 0)
255
+ return;
256
+ const seen = new Set();
257
+ snapshot.covers.forEach((target, targetIndex) => {
258
+ const targetRef = normalizeCoverageRef(target);
259
+ if (!targetRef && target.targetKind !== 'skill' && target.targetKind !== 'frontmatter')
260
+ return;
261
+ for (const artifact of skillArtifacts) {
262
+ const stableKey = coverageTargetStableKey(target, artifact.artifactHash);
263
+ const dedupeKey = `${sampleNodeId}|${stableKey}`;
264
+ if (seen.has(dedupeKey))
265
+ continue;
266
+ seen.add(dedupeKey);
267
+ const evidenceRefs = coverageEvidence(report, sampleId, targetIndex, target);
268
+ const targetNodeId = addNode(stableKey, COVERAGE_TARGET_NODE_KIND[target.targetKind], 'entity', coverageTargetLabel(target), {
269
+ binding: { bindingStrength: 'content-hash', keys: { artifactHash: artifact.artifactHash } },
270
+ attrs: {
271
+ display: {
272
+ targetKind: target.targetKind,
273
+ ref: targetRef,
274
+ variant: artifact.variant,
275
+ sourceLocator: artifact.config.locator,
276
+ },
277
+ },
278
+ evidenceRefs,
279
+ });
280
+ addEdge(sampleNodeId, targetNodeId, 'covers', {
281
+ confidence: 1,
282
+ binding: {
283
+ bindingStrength: 'explicit',
284
+ keys: {
285
+ sampleId,
286
+ targetKind: target.targetKind,
287
+ targetRef,
288
+ artifactHash: artifact.artifactHash,
289
+ },
290
+ },
291
+ attrs: { producer: { source: 'sample.covers' } },
292
+ evidenceRefs,
293
+ });
294
+ }
295
+ });
296
+ };
178
297
  const variantNodeIds = new Map();
179
298
  for (const variant of report.meta.variants) {
180
299
  const artifactHash = report.meta.artifactHashes?.[variant];
@@ -241,6 +360,7 @@ export function buildEvalArtifactGraph(options) {
241
360
  });
242
361
  addEdge(sampleNodeId, assertionNodeId, 'contains');
243
362
  });
363
+ addCoverageEdges(sampleId, sampleNodeId, snapshot);
244
364
  }
245
365
  for (const [resultIndex, result] of report.results.entries()) {
246
366
  const sampleNodeId = addNode(sampleStableKey(report, result.sample_id), 'sample', 'entity', result.sample_id, {
@@ -121,11 +121,11 @@ omk sample --batch
121
121
  ### 体检 skill 写法
122
122
 
123
123
  ```bash
124
- # 全量体检(含 LLM-judge 维度)
124
+ # 健康度审计(默认 --repeat 2 采样 + k/n 共识归并)
125
125
  omk doctor
126
126
 
127
- # 只跑静态检查不调 LLM
128
- omk doctor --static-only
127
+ # 单次快检(不采样、不归并,最省)
128
+ omk doctor --repeat 1
129
129
 
130
130
  # 针对单 skill
131
131
  omk doctor skills/my-skill.md
@@ -6,7 +6,7 @@
6
6
 
7
7
  ## omk doctor
8
8
 
9
- 体检 omk 工作目录,检查 skill 配置 / 依赖 / executor 连通性。
9
+ 体检 omk 工作目录:先跑静态规则,再对 skill 做多维度 LLM 健康度审计(默认 --repeat 2 采样 + 共识归并)。
10
10
 
11
11
  **用法:**
12
12
 
@@ -20,6 +20,7 @@ omk doctor [target] [flags]
20
20
 
21
21
  **Flags:**
22
22
 
23
+ - `--concurrency` `option`:多次采样的并发数。默认 = --repeat(全并行,各遍相互独立,压墙钟时间)。设 1 = 串行。成本不变,只抬高瞬时并发(rate-limit 敏感时调小)。
23
24
  - `--dimensions` `option`:自定义维度配置文件(YAML),追加到内置 7 维度之后。每条维度二选一:promptSection(走 LLM 体检)或 endpoint(POST skill 快照给接口判定)。注意:endpoint 会把 SKILL.md 全文 + 子文件发到该地址,仅对可信配置/可信地址启用。
24
25
  - `--effort` `option`:LLM 推理 effort:low / medium / high / xhigh / max。
25
26
  - `--executor` `option`:执行器名,默认 claude。指定为测试 fixture 路径可在测试里跑(同 omk doctor)。
@@ -30,25 +31,31 @@ omk doctor [target] [flags]
30
31
  - `--lang` `option` (默认 `zh`):输出语言 zh|en,优先级 CLI > OMK_LANG env > zh。
31
32
  - `--model` `option`:LLM model 名,默认 sonnet。
32
33
  - `--output-dir` `option`:报告输出目录,默认项目级 .omk/doctors(--global 写全局)。
33
- - `--samples` `option`:用例文件路径(.json/.yaml)。不传则按 target / cwd 顺序自动发现。
34
- - `--static-only` `boolean`:离线静态模式,只跑 4 条静态 rule(skill_readable / skill_metadata / dependencies_present / samples_contract_aligned),不调 LLM
34
+ - `--repeat` `option`:健康度体检重复采样次数(self-consistency)。默认 2:并行跑 2 遍、finding 取并集并用 LLM 聚类归并同根因、标注支持度 k/N,压低单次采样方差。设 1 = 单次快速体检(不采样、不归并,最省)。
35
+ - `--static-only` `boolean`:只跑静态检测(不调 LLM、不读 samples.json):skill 可读性 / frontmatter 合法性 / 正文引用的脚本·CLI·文件·env 是否存在。CI LLM 凭证或断网时用。
35
36
  - `--timeout` `option`:单次 LLM 会话超时秒数,默认 600(10 分钟)。
36
37
 
37
38
  **示例:**
38
39
 
39
- > 默认模式跑 LLM 健康度审计(7 内置维度)。
40
+ > 默认模式跑静态规则 + LLM 健康度审计(7 内置维度)。
40
41
 
41
42
  ```bash
42
43
  omk doctor
43
44
  ```
44
45
 
45
- > 离线静态模式,只跑 4 条静态 rule,不调 LLM,CI 无 LLM 凭证时用。
46
+ > 单次快速体检(不采样、不归并,最省)。
47
+
48
+ ```bash
49
+ omk doctor --repeat 1
50
+ ```
51
+
52
+ > 只跑静态检测(不调 LLM、不读 samples):结构 + 正文依赖检查。
46
53
 
47
54
  ```bash
48
55
  omk doctor --static-only
49
56
  ```
50
57
 
51
- > JSON 输出 + HTML 报告,给 CI 抓 exit code 同时人看。
58
+ > JSON 输出 + 静默 gate,给 CI 抓 exit code 同时人看。
52
59
 
53
60
  ```bash
54
61
  omk doctor --json --gate
@@ -14,14 +14,15 @@ export default class Doctor extends BaseCommand {
14
14
  gate: import("@oclif/core/interfaces").BooleanFlag<boolean>;
15
15
  executor: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
16
16
  model: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
17
- samples: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
18
17
  timeout: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
19
18
  'output-dir': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
20
19
  global: import("@oclif/core/interfaces").BooleanFlag<boolean>;
21
20
  dimensions: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
22
- 'static-only': import("@oclif/core/interfaces").BooleanFlag<boolean>;
23
21
  fix: import("@oclif/core/interfaces").BooleanFlag<boolean>;
24
22
  effort: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
23
+ repeat: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
24
+ concurrency: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
25
+ 'static-only': import("@oclif/core/interfaces").BooleanFlag<boolean>;
25
26
  };
26
27
  run(): Promise<void>;
27
28
  }
@@ -3,7 +3,7 @@ import { join, resolve } from 'node:path';
3
3
  import { Args, Flags } from '@oclif/core';
4
4
  import { LANG_FLAG, bilingual } from '../oclif/i18n.js';
5
5
  import { BaseCommand } from '../oclif/base-command.js';
6
- import { numberStringParser } from '../oclif/parsers.js';
6
+ import { enumStringParser, integerStringParser, numberStringParser } from '../oclif/parsers.js';
7
7
  import { CliExit } from '../lib/cli-exit.js';
8
8
  import { tCli } from '../lib/i18n.js';
9
9
  import { makeDoctorProgress } from '../lib/progress.js';
@@ -12,32 +12,38 @@ import { indexDoctorWrite, removeDoctorCard } from '../../eval-core/artifact-ind
12
12
  import { doctorReportFileStem, isReportFileName, reportFilePath } from '../../eval-core/artifact-file-names.js';
13
13
  import { migrateLegacyReportFiles } from '../../eval-core/report-file-migration.js';
14
14
  import { projectDoctorsDir, globalDoctorsDir } from '../../eval-core/measurement-dirs.js';
15
- import { findDoctorDeprecatedSamplesHint, findDoctorSamplesPath } from '../../inputs/sample-locator.js';
16
15
  import { persistDoctorGraphSidecars, removeDoctorGraphSidecars } from '../../artifact-graph/doctor.js';
17
16
  export default class Doctor extends BaseCommand {
18
17
  static description = bilingual({
19
- zh: '体检 omk 工作目录,检查 skill 配置 / 依赖 / executor 连通性。',
20
- en: 'Preflight health checks for omk workdir: skill config / deps / executor connectivity.',
18
+ zh: '体检 omk 工作目录:先跑静态规则,再对 skill 做多维度 LLM 健康度审计(默认 --repeat 2 采样 + 共识归并)。',
19
+ en: 'Preflight health checks for omk workdir: static rules plus multi-dimension LLM health audit of your skills (default --repeat 2 sampling + consensus merge).',
21
20
  });
22
21
  static examples = [
23
22
  {
24
23
  description: bilingual({
25
- zh: '默认模式跑 LLM 健康度审计(7 内置维度)。',
26
- en: 'Default mode runs LLM-driven health audit (7 built-in dimensions).',
24
+ zh: '默认模式跑静态规则 + LLM 健康度审计(7 内置维度)。',
25
+ en: 'Default mode runs static rules plus LLM-driven health audit (7 built-in dimensions).',
27
26
  }),
28
27
  command: '<%= config.bin %> doctor',
29
28
  },
30
29
  {
31
30
  description: bilingual({
32
- zh: '离线静态模式,只跑 4 条静态 rule,不调 LLM,CI 无 LLM 凭证时用。',
33
- en: 'Offline static mode, only 4 static rules, no LLM call. Use when CI lacks LLM credentials.',
31
+ zh: '单次快速体检(不采样、不归并,最省)。',
32
+ en: 'Single quick pass (no sampling/merge, cheapest).',
33
+ }),
34
+ command: '<%= config.bin %> doctor --repeat 1',
35
+ },
36
+ {
37
+ description: bilingual({
38
+ zh: '只跑静态检测(不调 LLM、不读 samples):结构 + 正文依赖检查。',
39
+ en: 'Static checks only (no LLM, no samples): structural + body-dependency checks.',
34
40
  }),
35
41
  command: '<%= config.bin %> doctor --static-only',
36
42
  },
37
43
  {
38
44
  description: bilingual({
39
- zh: 'JSON 输出 + HTML 报告,给 CI 抓 exit code 同时人看。',
40
- en: 'JSON output + HTML report, for CI exit code + human review.',
45
+ zh: 'JSON 输出 + 静默 gate,给 CI 抓 exit code 同时人看。',
46
+ en: 'JSON output + silent gate, for CI exit code + human review.',
41
47
  }),
42
48
  command: '<%= config.bin %> doctor --json --gate',
43
49
  },
@@ -79,12 +85,6 @@ export default class Doctor extends BaseCommand {
79
85
  en: 'LLM model name, default sonnet.',
80
86
  }),
81
87
  }),
82
- samples: Flags.string({
83
- description: bilingual({
84
- zh: '用例文件路径(.json/.yaml)。不传则按 target / cwd 顺序自动发现。',
85
- en: 'Samples file path (.json/.yaml). Auto-detects from target / cwd if omitted.',
86
- }),
87
- }),
88
88
  timeout: Flags.string({
89
89
  description: bilingual({
90
90
  zh: '单次 LLM 会话超时秒数,默认 600(10 分钟)。',
@@ -110,13 +110,6 @@ export default class Doctor extends BaseCommand {
110
110
  en: 'Custom dimensions config file (YAML), appended after builtin 7. Each is either promptSection (LLM audit) or endpoint (POST skill snapshot to your service). Note: endpoint sends the full SKILL.md + sub-files to that URL — only enable for trusted configs/URLs.',
111
111
  }),
112
112
  }),
113
- 'static-only': Flags.boolean({
114
- description: bilingual({
115
- zh: '离线静态模式,只跑 4 条静态 rule(skill_readable / skill_metadata / dependencies_present / samples_contract_aligned),不调 LLM。',
116
- en: 'Offline static mode: only 4 static rules, no LLM call.',
117
- }),
118
- default: false,
119
- }),
120
113
  fix: Flags.boolean({
121
114
  description: bilingual({
122
115
  zh: '交互式修复:根据 doctor 报告问题,用 LLM agent 修复 skill。',
@@ -129,6 +122,28 @@ export default class Doctor extends BaseCommand {
129
122
  zh: 'LLM 推理 effort:low / medium / high / xhigh / max。',
130
123
  en: 'LLM reasoning effort: low / medium / high / xhigh / max.',
131
124
  }),
125
+ parse: enumStringParser('--effort', ['low', 'medium', 'high', 'xhigh', 'max']),
126
+ }),
127
+ repeat: Flags.string({
128
+ description: bilingual({
129
+ zh: '健康度体检重复采样次数(self-consistency)。默认 2:并行跑 2 遍、finding 取并集并用 LLM 聚类归并同根因、标注支持度 k/N,压低单次采样方差。设 1 = 单次快速体检(不采样、不归并,最省)。',
130
+ en: 'Health-check repeat count (self-consistency). Default 2: runs 2 passes in parallel, unions findings, merges same root cause via an LLM pass, tags k/N support. Set 1 for a single quick pass (no sampling/merge, cheapest).',
131
+ }),
132
+ parse: integerStringParser('--repeat', { min: 1, max: 10 }),
133
+ }),
134
+ concurrency: Flags.string({
135
+ description: bilingual({
136
+ zh: '多次采样的并发数。默认 = --repeat(全并行,各遍相互独立,压墙钟时间)。设 1 = 串行。成本不变,只抬高瞬时并发(rate-limit 敏感时调小)。',
137
+ en: 'Concurrency across the repeated passes. Default = --repeat (full parallel; passes are independent, cuts wall-clock). Set 1 for serial. Cost unchanged; only raises peak concurrency (lower it if rate-limited).',
138
+ }),
139
+ parse: integerStringParser('--concurrency', { min: 1, max: 10 }),
140
+ }),
141
+ 'static-only': Flags.boolean({
142
+ description: bilingual({
143
+ zh: '只跑静态检测(不调 LLM、不读 samples.json):skill 可读性 / frontmatter 合法性 / 正文引用的脚本·CLI·文件·env 是否存在。CI 无 LLM 凭证或断网时用。',
144
+ en: 'Static checks only (no LLM, no samples.json): readability / frontmatter / existence of scripts·CLI·files·env referenced in the skill body. For CI without LLM creds / offline.',
145
+ }),
146
+ default: false,
132
147
  }),
133
148
  };
134
149
  async run() {
@@ -138,39 +153,21 @@ export default class Doctor extends BaseCommand {
138
153
  const target = args.target ?? null;
139
154
  const executorName = flags.executor ?? 'claude';
140
155
  const model = flags.model ?? 'sonnet';
141
- // 默认 LLM 健康度审计(7 内置维度 + 用户注册的自定义维度);--static-only 切到
142
- // 离线静态模式:只跑 4 条静态 rule,不调 LLM。CI 节点没装 claude/codex、本地断网
143
- // 调试等场景。
144
- const staticOnly = flags['static-only'];
145
- const runHealthCheck = !staticOnly;
156
+ // omk doctor 默认 = 静态规则 + LLM 健康度审计(7 内置维度 + 用户注册的自定义维度);
157
+ // --static-only = 只跑静态检测(readable / metadata / 正文依赖,不调 LLM、不读 samples)
158
+ // samples_contract_aligned 仍只归 eval preflight(它要 samples.json,与离线解耦)。
159
+ // 健康度体检重复采样次数(CLI flag --repeat → 内部 healthSamples 字段):默认 2,
160
+ // 1 = 单次快检。合法范围由 oclif parser 拦截。
161
+ const healthSamples = flags.repeat != null ? Number(flags.repeat) : 2;
162
+ // 归并策略:CLI 恒用 llm(硬逻辑,不暴露开关);samples=1 时 composer 自动跳过归并。
163
+ // 失败回退 string 仍在 composer 内兜底。programmatic runDoctor 默认仍是 string。
164
+ const healthMerge = 'llm';
165
+ // 并发数(--concurrency → healthConcurrency):默认不传(composer 取 = healthSamples 全并行);显式 ≥1 才覆盖。
166
+ const healthConcurrency = flags.concurrency != null ? Number(flags.concurrency) : undefined;
146
167
  const defaultTimeoutSec = 600;
147
168
  const timeoutSec = flags.timeout != null ? Number(flags.timeout) : defaultTimeoutSec;
148
169
  const timeoutMs = Math.max(1000, Math.floor((Number.isFinite(timeoutSec) ? timeoutSec : defaultTimeoutSec) * 1000));
149
170
  const cwd = process.cwd();
150
- const samplesPath = flags.samples ? resolve(flags.samples) : findDoctorSamplesPath(target, cwd);
151
- const deprecatedSamplesHint = !flags.samples && !samplesPath
152
- ? findDoctorDeprecatedSamplesHint(target, cwd)
153
- : null;
154
- if (deprecatedSamplesHint) {
155
- process.stderr.write(tCli('cli.common.deprecated_skill_samples_path', lang, {
156
- oldPath: deprecatedSamplesHint.oldPath,
157
- newPath: deprecatedSamplesHint.newPath,
158
- }));
159
- }
160
- let samples;
161
- let requires;
162
- if (samplesPath) {
163
- try {
164
- const { loadSamples } = await import('../../inputs/load-samples.js');
165
- const loaded = loadSamples(samplesPath);
166
- samples = loaded.samples;
167
- requires = loaded.requires;
168
- }
169
- catch (err) {
170
- const msg = err instanceof Error ? err.message : String(err);
171
- process.stderr.write(tCli('cli.common.warn_load_samples_failed', lang, { path: samplesPath, message: msg }));
172
- }
173
- }
174
171
  // 副作用 import: 注册 7 内置维度 spec + skill_health composer rule。
175
172
  await import('../../doctor/health/register.js');
176
173
  if (flags.dimensions) {
@@ -184,15 +181,22 @@ export default class Doctor extends BaseCommand {
184
181
  const { renderDoctorReportText, renderDoctorReportJson } = await import('../../doctor/renderer.js');
185
182
  const { getRegisteredRules } = await import('../../doctor/rules.js');
186
183
  const { isComposerRule } = await import('../../types/doctor.js');
187
- // 在线检查(LLM health composer + endpoint 自定义维度)默认跑;--static-only
188
- // 离线模式只跑纯静态的内置 rule(无网络 / LLM)。endpoint rule 标了 external=true。
184
+ // 默认:静态规则 + 在线检查(LLM health composer + endpoint 自定义维度 external=true)
185
+ // --static-only:只跑静态检测(纯静态内置 rule),但排除 samples_contract_aligned
186
+ // (那条要 samples.json,与离线解耦) → 且不加载 samples,依赖检查只扫 skill 正文。
187
+ const staticOnly = flags['static-only'];
189
188
  const isOnline = (r) => isComposerRule(r) || r.external === true;
190
189
  const rulesOverride = staticOnly
191
- ? getRegisteredRules().filter((r) => !isOnline(r))
192
- : getRegisteredRules().filter(isOnline);
190
+ ? getRegisteredRules().filter((r) => !isOnline(r) && r.id !== 'samples_contract_aligned')
191
+ : getRegisteredRules().filter((r) => r.id !== 'samples_contract_aligned');
193
192
  // 批量体检进度(per-skill,写 stderr)。--gate 是静默模式,不报进度;
194
193
  // --json 进度走 stderr 不污染 stdout 的 JSON。
195
194
  const onProgress = flags.gate ? undefined : makeDoctorProgress(lang);
195
+ // --effort 合法性由 oclif parser 拦截。这里仅做窄化后透传给健康审计。
196
+ const validEfforts = ['low', 'medium', 'high', 'xhigh', 'max'];
197
+ const effort = flags.effort && validEfforts.includes(flags.effort)
198
+ ? flags.effort
199
+ : undefined;
196
200
  let report;
197
201
  try {
198
202
  report = await runDoctor({
@@ -202,10 +206,12 @@ export default class Doctor extends BaseCommand {
202
206
  model,
203
207
  timeoutMs,
204
208
  lang,
205
- runHealthCheck,
209
+ runHealthCheck: !staticOnly,
210
+ healthSamples,
211
+ healthMerge,
212
+ healthConcurrency,
213
+ effort,
206
214
  rules: rulesOverride,
207
- samples,
208
- requires,
209
215
  onProgress,
210
216
  });
211
217
  }
@@ -233,9 +239,6 @@ export default class Doctor extends BaseCommand {
233
239
  }
234
240
  }
235
241
  else {
236
- if (samplesPath && samples) {
237
- process.stderr.write(tCli('cli.doctor.samples_detected', lang, { path: samplesPath }) + '\n');
238
- }
239
242
  renderDoctorReportText(report, lang);
240
243
  }
241
244
  persistDoctorReport(report, flags['output-dir']
@@ -248,7 +251,7 @@ export default class Doctor extends BaseCommand {
248
251
  throw new CliExit(0);
249
252
  }
250
253
  const { runDoctorFix } = await import('../../doctor/fixer.js');
251
- const changed = await runDoctorFix({ report: existing, executorName, model, timeoutMs });
254
+ const changed = await runDoctorFix({ report: existing, executorName, model, timeoutMs, effort });
252
255
  throw new CliExit(changed ? 0 : (existing.outcome === 'failed' ? 1 : 0));
253
256
  }
254
257
  throw new CliExit(report.outcome === 'failed' ? 1 : 0);
@@ -58,11 +58,12 @@ export interface DoctorFlags {
58
58
  gate: boolean;
59
59
  executor?: string;
60
60
  model?: string;
61
- samples?: string;
62
61
  timeout?: string;
63
62
  'output-dir'?: string;
64
63
  dimensions?: string;
65
- 'static-only': boolean;
64
+ repeat?: string;
65
+ concurrency?: string;
66
+ effort?: string;
66
67
  }
67
68
  export type EvalArgs = Record<string, never>;
68
69
  export interface EvalFlags {
@@ -1,3 +1,3 @@
1
1
  import type { CliMessage } from './types.js';
2
- export type CommonMessageKey = 'cli.common.unknown_domain' | 'cli.common.error_prefix' | 'cli.common.skill_dir_not_found' | 'cli.common.skill_file_not_found' | 'cli.common.skill_dir_no_skill_md' | 'cli.common.report_not_found' | 'cli.common.no_judge_model' | 'cli.common.judge_models_single_only' | 'cli.common.warn_load_samples_failed' | 'cli.common.deprecated_skill_samples_path' | 'cli.common.samples_not_found' | 'cli.update.new_version_available' | 'cli.update.box_title' | 'cli.update.box_version_line' | 'cli.update.box_upgrade_line' | 'cli.update.box_silence_line' | 'cli.observe.view_hint' | 'cli.observe.observation_recorded' | 'cli.observe.production_gap' | 'cli.studio.started' | 'cli.studio.stop_hint' | 'cli.studio.open_failed' | 'cli.doctor.no_skill_found' | 'cli.doctor.samples_detected' | 'cli.doctor.progress_skill_start' | 'cli.doctor.progress_skill_done';
2
+ export type CommonMessageKey = 'cli.common.unknown_domain' | 'cli.common.error_prefix' | 'cli.common.skill_dir_not_found' | 'cli.common.skill_file_not_found' | 'cli.common.skill_dir_no_skill_md' | 'cli.common.report_not_found' | 'cli.common.no_judge_model' | 'cli.common.judge_models_single_only' | 'cli.common.warn_load_samples_failed' | 'cli.common.deprecated_skill_samples_path' | 'cli.common.samples_not_found' | 'cli.update.new_version_available' | 'cli.update.box_title' | 'cli.update.box_version_line' | 'cli.update.box_upgrade_line' | 'cli.update.box_silence_line' | 'cli.observe.view_hint' | 'cli.observe.observation_recorded' | 'cli.observe.production_gap' | 'cli.studio.started' | 'cli.studio.stop_hint' | 'cli.studio.open_failed' | 'cli.doctor.no_skill_found' | 'cli.doctor.progress_skill_start' | 'cli.doctor.progress_skill_done';
3
3
  export declare const commonDict: Record<CommonMessageKey, CliMessage>;
@@ -91,10 +91,6 @@ export const commonDict = {
91
91
  zh: '未在 {path} 下发现 skill 文件。\n doctor 期望 .md 文件、目录(包含 .md 或 SKILL.md)或 cwd 下的 skills/ 子目录。',
92
92
  en: 'No skills found at {path}.\n doctor expects a .md file, a directory (containing .md or SKILL.md), or skills/ under cwd.',
93
93
  },
94
- 'cli.doctor.samples_detected': {
95
- zh: '✓ 使用评测用例文件:{path}',
96
- en: '✓ Using eval samples file: {path}',
97
- },
98
94
  'cli.doctor.progress_skill_start': {
99
95
  zh: '{prefix}{skill} ⏳ 体检中...\n',
100
96
  en: '{prefix}{skill} ⏳ checking...\n',
@@ -222,7 +222,7 @@ export function makeEndpointRule(spec, fetchFn = fetch) {
222
222
  severity: spec.severity,
223
223
  // 自定义 key 不在 DOCTOR_MESSAGES 字典里,renderer 会 fallback 到 ruleId(=spec.id)。
224
224
  labelKey: `cli.doctor.endpoint.${spec.id}`,
225
- // 网络检查,与 health composer 同档:默认跑,--static-only 跳过。
225
+ // 网络检查,与 health composer 同档:CLI doctor 默认跑(在线检查)。
226
226
  external: true,
227
227
  async check(ctx) {
228
228
  // SSRF 防护(defense-in-depth):endpoint 会收到 skill 完整快照,且响应