oh-my-knowledge 0.42.0 → 0.44.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/README.md +1 -1
  2. package/README.zh.md +1 -1
  3. package/dist/artifact-graph/doctor.d.ts +21 -0
  4. package/dist/artifact-graph/doctor.js +569 -0
  5. package/dist/artifact-graph/eval.d.ts +16 -0
  6. package/dist/artifact-graph/eval.js +351 -0
  7. package/dist/assets/agent-skills/omk/SKILL.md +9 -9
  8. package/dist/assets/agent-skills/omk/references/commands.md +1 -1
  9. package/dist/cli/commands/doctor.js +62 -61
  10. package/dist/cli/commands/eval/index.js +20 -2
  11. package/dist/cli/commands/init.js +1 -0
  12. package/dist/cli/commands/observe/index.d.ts +2 -2
  13. package/dist/cli/commands/observe/index.js +8 -7
  14. package/dist/cli/commands/sample.js +22 -16
  15. package/dist/cli/lib/i18n-dict/common.d.ts +1 -1
  16. package/dist/cli/lib/i18n-dict/common.js +8 -0
  17. package/dist/cli/lib/i18n-dict/help.js +12 -10
  18. package/dist/cli/lib/parse-run-config/samples-discovery.d.ts +5 -7
  19. package/dist/cli/lib/parse-run-config/samples-discovery.js +10 -32
  20. package/dist/cli/lib/parse-run-config.js +4 -4
  21. package/dist/cli/lib/resolve-skill-input.js +10 -12
  22. package/dist/doctor/index.js +2 -7
  23. package/dist/doctor/messages.js +2 -2
  24. package/dist/eval-core/artifact-file-names.d.ts +15 -0
  25. package/dist/eval-core/artifact-file-names.js +45 -0
  26. package/dist/eval-core/evaluation-reporting.d.ts +1 -1
  27. package/dist/eval-core/evaluation-reporting.js +32 -12
  28. package/dist/eval-core/measurement-dirs.js +13 -7
  29. package/dist/eval-core/report-file-migration.d.ts +10 -0
  30. package/dist/eval-core/report-file-migration.js +90 -0
  31. package/dist/inputs/sample-locator.d.ts +23 -0
  32. package/dist/inputs/sample-locator.js +195 -0
  33. package/dist/inputs/skill-loader.js +7 -17
  34. package/dist/observability/inbox.js +7 -3
  35. package/dist/renderer/html-renderer.js +3 -2
  36. package/dist/renderer/skill-detail-renderer.js +832 -0
  37. package/dist/renderer/skill-list-renderer.js +2 -7
  38. package/dist/server/report-server.js +55 -15
  39. package/dist/server/report-store.js +17 -9
  40. package/dist/server/skill-index.d.ts +6 -0
  41. package/dist/server/skill-index.js +399 -15
  42. package/dist/types/artifact-graph.d.ts +93 -0
  43. package/dist/types/artifact-graph.js +1 -0
  44. package/dist/types/index.d.ts +1 -0
  45. package/dist/types/index.js +1 -0
  46. package/dist/types/skill-index.d.ts +38 -0
  47. package/package.json +1 -1
@@ -2,8 +2,8 @@ import { BaseCommand } from '../../oclif/base-command.js';
2
2
  import type { SkillHealthReport } from '../../../observability/skill-health-analyzer.js';
3
3
  /**
4
4
  * observe-health 报告落盘:id / 文件名加 4 位随机段,根治「同秒两次 omk observe 直接覆盖、数据丢失」的 bug。
5
- * 保留 `-observe-health.json` 后缀 —— resolveObserveHealthDir 靠它判项目优先、listAnalyses 靠它取 id stem、
6
- * loadAnalysis 靠 `${id}.json` 读真身。落盘后 best-effort 追加全局轻卡片,让 studio 跨项目聚合。
5
+ * 文件名统一为 `{run}.report.json`:目录表达 observe-health 域,文件表达这是可读报告。
6
+ * 落盘后 best-effort 追加全局轻卡片,让 studio 跨项目聚合。
7
7
  */
8
8
  export declare function persistObserveHealthReport(report: SkillHealthReport, outDir: string): {
9
9
  id: string;
@@ -1,5 +1,5 @@
1
1
  import { mkdirSync, writeFileSync } from 'node:fs';
2
- import { resolve, join } from 'node:path';
2
+ import { resolve } from 'node:path';
3
3
  import { Args, Flags } from '@oclif/core';
4
4
  import { LANG_FLAG, bilingual } from '../../oclif/i18n.js';
5
5
  import { BaseCommand } from '../../oclif/base-command.js';
@@ -8,17 +8,18 @@ import { tCli } from '../../lib/i18n.js';
8
8
  import { parseLastWindow } from '../../lib/shared.js';
9
9
  import { projectObserveHealthDir, globalObserveHealthDir } from '../../../eval-core/measurement-dirs.js';
10
10
  import { indexObserveWrite } from '../../../eval-core/artifact-index.js';
11
+ import { reportFilePath, runFileSuffix } from '../../../eval-core/artifact-file-names.js';
12
+ import { migrateLegacyReportFiles } from '../../../eval-core/report-file-migration.js';
11
13
  /**
12
14
  * observe-health 报告落盘:id / 文件名加 4 位随机段,根治「同秒两次 omk observe 直接覆盖、数据丢失」的 bug。
13
- * 保留 `-observe-health.json` 后缀 —— resolveObserveHealthDir 靠它判项目优先、listAnalyses 靠它取 id stem、
14
- * loadAnalysis 靠 `${id}.json` 读真身。落盘后 best-effort 追加全局轻卡片,让 studio 跨项目聚合。
15
+ * 文件名统一为 `{run}.report.json`:目录表达 observe-health 域,文件表达这是可读报告。
16
+ * 落盘后 best-effort 追加全局轻卡片,让 studio 跨项目聚合。
15
17
  */
16
18
  export function persistObserveHealthReport(report, outDir) {
17
19
  mkdirSync(outDir, { recursive: true });
18
- const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
19
- const rand = Math.random().toString(36).slice(2, 6);
20
- const id = `${timestamp}-${rand}-observe-health`;
21
- const jsonPath = join(outDir, `${id}.json`);
20
+ migrateLegacyReportFiles(outDir, 'observe-health');
21
+ const id = runFileSuffix();
22
+ const jsonPath = reportFilePath(outDir, id);
22
23
  writeFileSync(jsonPath, JSON.stringify(report, null, 2));
23
24
  indexObserveWrite(report, jsonPath, outDir, id);
24
25
  return { id, jsonPath };
@@ -9,6 +9,7 @@ import { CliExit } from '../lib/cli-exit.js';
9
9
  import { tCli } from '../lib/i18n.js';
10
10
  import { projectReportsDir, globalReportsDir } from '../../eval-core/measurement-dirs.js';
11
11
  import { loadSamples, parseYaml, listSampleFilesInDir } from '../../inputs/load-samples.js';
12
+ import { defaultFlatSkillSamplesFile, defaultSkillLocalSamplesFile, findFlatSkillSamplesPath, findSkillSamplesPath, } from '../../inputs/sample-locator.js';
12
13
  import { hashSample } from '../../eval-core/evaluation-reporting.js';
13
14
  import { hashArtifactSource } from '../../inputs/content-hash.js';
14
15
  function isRecord(value) {
@@ -233,23 +234,23 @@ async function runSampleFix(args, flags, lang) {
233
234
  console.error(lang === 'zh' ? '请指定 skill 路径,如: omk sample skills/my-skill/SKILL.md --fix' : 'Specify skill path: omk sample skills/my-skill/SKILL.md --fix');
234
235
  throw new CliExit(1);
235
236
  }
236
- const resolvedSkillPath = resolve(skillPath);
237
- if (!existsSync(resolvedSkillPath)) {
238
- console.error(lang === 'zh' ? `skill 文件不存在: ${resolvedSkillPath}` : `Skill file not found: ${resolvedSkillPath}`);
237
+ const { resolveSkillInput } = await import('../lib/resolve-skill-input.js');
238
+ let resolvedInput;
239
+ try {
240
+ resolvedInput = resolveSkillInput(skillPath, lang);
241
+ }
242
+ catch (err) {
243
+ console.error(err instanceof Error ? err.message : String(err));
239
244
  throw new CliExit(1);
240
245
  }
241
- const isDir = basename(resolvedSkillPath) === 'SKILL.md';
242
- const skillDir = isDir ? dirname(resolvedSkillPath) : dirname(resolvedSkillPath);
243
- const samplesInput = isDir
244
- ? join(skillDir, '.omk')
245
- : resolve('eval-samples.json');
246
+ const samplesInput = resolvedInput.samplesPath;
246
247
  if (!existsSync(samplesInput)) {
247
248
  console.error(lang === 'zh' ? `samples 路径不存在: ${samplesInput},先运行 omk sample 生成` : `Samples path not found: ${samplesInput}, run omk sample first`);
248
249
  throw new CliExit(1);
249
250
  }
250
- const defaultTreatmentName = isDir
251
- ? basename(skillDir)
252
- : basename(resolvedSkillPath, extname(resolvedSkillPath));
251
+ const defaultTreatmentName = resolvedInput.isDirectorySkill
252
+ ? basename(resolvedInput.skillDir)
253
+ : basename(resolvedInput.skillPath, extname(resolvedInput.skillPath));
253
254
  const treatmentName = flags.treatment ?? defaultTreatmentName;
254
255
  process.stderr.write(lang === 'zh' ? `🔍 正在查找 ${treatmentName} 的最新评测报告...\n` : `🔍 Scanning latest report for ${treatmentName}...\n`);
255
256
  // 显式 --reports-dir 固定该目录;默认 overlay(项目 .omk/reports 盖全局),findByVariant 记录优先看项目、
@@ -277,7 +278,7 @@ async function runSampleFix(args, flags, lang) {
277
278
  throw new CliExit(1);
278
279
  }
279
280
  const samples = loadedSamples.samples;
280
- const skillContent = readFileSync(resolvedSkillPath, 'utf-8');
281
+ const skillContent = readFileSync(resolvedInput.skillPath, 'utf-8');
281
282
  const sampleDesignIds = collectSampleDesignFailureIds(report, treatmentName);
282
283
  const sampleDesignCount = sampleDesignIds.size;
283
284
  if (sampleDesignCount === 0) {
@@ -286,7 +287,7 @@ async function runSampleFix(args, flags, lang) {
286
287
  }
287
288
  // 当前内容指纹走整树哈,与 eval 报告口径一致:dir-skill(用户传 .../SKILL.md)哈整棵 skill 目录、
288
289
  // 单文件 .md 哈单文件字节。
289
- const currentContentHash = hashArtifactSource(isDir ? skillDir : resolvedSkillPath, isDir);
290
+ const currentContentHash = hashArtifactSource(resolvedInput.isDirectorySkill ? resolvedInput.skillDir : resolvedInput.skillPath, resolvedInput.isDirectorySkill);
290
291
  try {
291
292
  assertFixReportMatchesCurrentInputs({
292
293
  report,
@@ -435,24 +436,29 @@ async function runSample(args, flags, lang) {
435
436
  let name;
436
437
  let skillPath;
437
438
  let samplesPath;
439
+ let existingSamplesPath;
438
440
  const fullPath = join(skillDir, entry);
439
441
  if (entry.endsWith('.md') && !entry.endsWith('.eval-samples.json')) {
440
442
  name = entry.slice(0, -3);
441
443
  skillPath = fullPath;
442
- samplesPath = join(skillDir, `${name}.eval-samples.json`);
444
+ samplesPath = defaultFlatSkillSamplesFile(skillDir, name);
445
+ existingSamplesPath = findFlatSkillSamplesPath(skillDir, name);
443
446
  }
444
447
  else if (statSync(fullPath).isDirectory()) {
445
448
  const skillMd = join(fullPath, 'SKILL.md');
446
449
  if (!existsSync(skillMd))
447
450
  continue;
451
+ if (existsSync(join(skillDir, `${entry}.md`)))
452
+ continue;
448
453
  name = entry;
449
454
  skillPath = skillMd;
450
- samplesPath = join(fullPath, '.omk', 'samples.json');
455
+ samplesPath = defaultSkillLocalSamplesFile(fullPath);
456
+ existingSamplesPath = findSkillSamplesPath(fullPath);
451
457
  }
452
458
  else {
453
459
  continue;
454
460
  }
455
- if (existsSync(samplesPath)) {
461
+ if (existingSamplesPath) {
456
462
  process.stderr.write(tCli('cli.gen.skill_skipped_existing', lang, { name }));
457
463
  continue;
458
464
  }
@@ -1,3 +1,3 @@
1
1
  import type { CliMessage } from './types.js';
2
- export type CommonMessageKey = 'cli.common.unknown_domain' | 'cli.common.error_prefix' | 'cli.common.skill_dir_not_found' | 'cli.common.skill_file_not_found' | 'cli.common.skill_dir_no_skill_md' | 'cli.common.report_not_found' | 'cli.common.no_judge_model' | 'cli.common.judge_models_single_only' | 'cli.common.warn_load_samples_failed' | 'cli.update.new_version_available' | 'cli.update.box_title' | 'cli.update.box_version_line' | 'cli.update.box_upgrade_line' | 'cli.update.box_silence_line' | 'cli.observe.view_hint' | 'cli.observe.observation_recorded' | 'cli.observe.production_gap' | 'cli.studio.started' | 'cli.studio.stop_hint' | 'cli.studio.open_failed' | 'cli.doctor.no_skill_found' | 'cli.doctor.samples_detected' | 'cli.doctor.progress_skill_start' | 'cli.doctor.progress_skill_done';
2
+ export type CommonMessageKey = 'cli.common.unknown_domain' | 'cli.common.error_prefix' | 'cli.common.skill_dir_not_found' | 'cli.common.skill_file_not_found' | 'cli.common.skill_dir_no_skill_md' | 'cli.common.report_not_found' | 'cli.common.no_judge_model' | 'cli.common.judge_models_single_only' | 'cli.common.warn_load_samples_failed' | 'cli.common.deprecated_skill_samples_path' | 'cli.common.samples_not_found' | 'cli.update.new_version_available' | 'cli.update.box_title' | 'cli.update.box_version_line' | 'cli.update.box_upgrade_line' | 'cli.update.box_silence_line' | 'cli.observe.view_hint' | 'cli.observe.observation_recorded' | 'cli.observe.production_gap' | 'cli.studio.started' | 'cli.studio.stop_hint' | 'cli.studio.open_failed' | 'cli.doctor.no_skill_found' | 'cli.doctor.samples_detected' | 'cli.doctor.progress_skill_start' | 'cli.doctor.progress_skill_done';
3
3
  export declare const commonDict: Record<CommonMessageKey, CliMessage>;
@@ -35,6 +35,14 @@ export const commonDict = {
35
35
  zh: '⚠ 加载 samples 文件失败 ({path}): {message}\n',
36
36
  en: '⚠ Failed to load samples file ({path}): {message}\n',
37
37
  },
38
+ 'cli.common.deprecated_skill_samples_path': {
39
+ zh: '⚠ 发现旧的目录 skill 用例位置:{oldPath}。目录 skill 的私有用例已改放到 {newPath},请迁移。\n',
40
+ en: '⚠ Found deprecated directory skill samples path: {oldPath}. Directory skill samples now live at {newPath}; please migrate.\n',
41
+ },
42
+ 'cli.common.samples_not_found': {
43
+ zh: '未找到评测用例:{path}。请通过 --samples 指定文件,或创建项目级 eval-samples.json;单 treatment 目录 skill 请使用 <skill>/.omk/samples.json。',
44
+ en: 'Eval samples not found: {path}. Pass --samples, create project-level eval-samples.json, or use <skill>/.omk/samples.json for a single-treatment directory skill.',
45
+ },
38
46
  'cli.update.new_version_available': {
39
47
  zh: '\n💡 新版本可用:{old} → {new},运行 npm i -g oh-my-knowledge@latest 升级\n\n',
40
48
  en: '\n💡 New version available: {old} → {new}, run npm i -g oh-my-knowledge@latest to upgrade\n\n',
@@ -170,20 +170,21 @@ omk sample——生成或补齐 eval-samples 评测用例
170
170
  omk sample --batch [--skill-dir <dir>] [options]
171
171
 
172
172
  输出位置(默认):
173
- <skill>/SKILL.md → <skill>/.omk/samples.json(omk 标准约定)
174
- 其他 .md 路径 → 当前目录的 eval-samples.json(兜底)
173
+ 目录 skill(<skill>/SKILL.md) → <skill>/.omk/samples.json(omk 标准约定)
174
+ 扁平 .md(单次) → 当前目录的 eval-samples.json(项目级兜底)
175
+ 扁平 .md(--batch) → <skill-dir>/<name>.eval-samples.json(兼容 paired 布局)
175
176
 
176
177
  选项:
177
178
  --count <n> 强制生成 N 条(不指定时由 LLM 按 skill 类型自动判断:
178
179
  工作流型 6-8 条 / 原子型 4-6 条 / 混合型 5-7 条)
179
180
  --model <name> 生成模型(默认:opus;lean+effort-low 已自动开,想省钱可改 sonnet/haiku)
180
181
  --focus <text> 自然语言指定希望覆盖的场景(追加到 prompt,优先级高于自由发挥)
181
- --batch 为 skill 目录下缺少 eval-samples 的 skill 批量生成
182
+ --batch 为 skill 目录下缺少 samples 的 skill 批量生成
182
183
  --skill-dir <path> skill 目录(batch 使用,默认:skills)
183
184
 
184
185
  示例:
185
- omk improve samples skills/req-tool.md
186
- omk improve samples skills/req-tool.md --count 8 \\
186
+ omk sample skills/req-tool.md
187
+ omk sample skills/req-tool.md --count 8 \\
187
188
  --focus "重点覆盖 tag 查询走 PROJECT 空 → WORKSPACE 兜底的多步流程,以及 search 失败的错误路径"
188
189
  `,
189
190
  en: `
@@ -194,20 +195,21 @@ Usage:
194
195
  omk sample --batch [--skill-dir <dir>] [options]
195
196
 
196
197
  Output path (default):
197
- <skill>/SKILL.md → <skill>/.omk/samples.json (omk standard layout)
198
- other .md paths → ./eval-samples.json in current directory (fallback)
198
+ directory skill (<skill>/SKILL.md) → <skill>/.omk/samples.json (omk standard layout)
199
+ flat .md (single) → ./eval-samples.json in current directory (project fallback)
200
+ flat .md (--batch) → <skill-dir>/<name>.eval-samples.json (compatible paired layout)
199
201
 
200
202
  Options:
201
203
  --count <n> Force N samples (omit to let LLM auto-decide by skill type:
202
204
  workflow 6-8 / atomic 4-6 / mixed 5-7)
203
205
  --model <name> Generation model (default: opus; lean+effort-low applied; pass --model sonnet/haiku to save cost)
204
206
  --focus <text> Natural-language scenario hints appended to the prompt (overrides freeform diversity)
205
- --batch Generate for skills that are missing eval-samples
207
+ --batch Generate for skills that are missing samples
206
208
  --skill-dir <path> Skill directory for batch mode (default: skills)
207
209
 
208
210
  Examples:
209
- omk improve samples skills/req-tool.md
210
- omk improve samples skills/req-tool.md --count 8 \\
211
+ omk sample skills/req-tool.md
212
+ omk sample skills/req-tool.md --count 8 \\
211
213
  --focus "Cover PROJECT-empty → WORKSPACE-fallback multi-step tag lookup and the search-failure error path"
212
214
  `,
213
215
  },
@@ -1,18 +1,16 @@
1
1
  /**
2
2
  * 当 CLI 未传 `--samples` 时,自动发现 sample 路径。
3
3
  *
4
- * 发现顺序(单 treatment 模式):
5
- * 1. treatment 是绝对 / 相对路径(文件 / 目录) → 找它所在目录的 `.omk/`
6
- * 2. fallback 找目录里的 `eval-samples.{json,yaml,yml}`
7
- * 3. fallback `<skillDir>/<treatmentName>/.omk/`
8
- * 4. 终极兜底 cwd 下的 `eval-samples.{json,yaml,yml}`
4
+ * 发现顺序:
5
+ * 1. 单 treatment → 找该 skill 私有 samples(`<skill>/.omk/`)或扁平 skill paired 文件
6
+ * 2. 终极兜底 cwd 下的项目级 `eval-samples.{json,yaml,yml}`
9
7
  *
10
8
  * `loadSamples` 在下游自己处理「文件 vs 目录」—— 目录模式 glob `*.{json,yaml,yml}`
11
9
  * 合并,跳过 reserved 前缀(`report-` / `health-` / `_`)。所以一个 skill 可以把
12
10
  * sample 拆到多文件(`workflow.json` + `platform.json`),也可以单 `samples.json`,
13
11
  * 两种都行。
14
12
  *
15
- * 多 treatment 评测必须显式传 `--samples`(因为「找哪个 skill 的 bundled samples」
16
- * 没有唯一答案)。本函数仅处理单 treatment 与零 treatment 情况。
13
+ * 多 treatment 评测不会触发 skill-local 自动发现(因为「找哪个 skill 的 bundled samples」
14
+ * 没有唯一答案),只回到项目级 samples 兜底。
17
15
  */
18
16
  export declare function discoverSamplesPath(values: Record<string, unknown>, skillDir: string): string;
@@ -1,50 +1,28 @@
1
1
  /**
2
2
  * 当 CLI 未传 `--samples` 时,自动发现 sample 路径。
3
3
  *
4
- * 发现顺序(单 treatment 模式):
5
- * 1. treatment 是绝对 / 相对路径(文件 / 目录) → 找它所在目录的 `.omk/`
6
- * 2. fallback 找目录里的 `eval-samples.{json,yaml,yml}`
7
- * 3. fallback `<skillDir>/<treatmentName>/.omk/`
8
- * 4. 终极兜底 cwd 下的 `eval-samples.{json,yaml,yml}`
4
+ * 发现顺序:
5
+ * 1. 单 treatment → 找该 skill 私有 samples(`<skill>/.omk/`)或扁平 skill paired 文件
6
+ * 2. 终极兜底 cwd 下的项目级 `eval-samples.{json,yaml,yml}`
9
7
  *
10
8
  * `loadSamples` 在下游自己处理「文件 vs 目录」—— 目录模式 glob `*.{json,yaml,yml}`
11
9
  * 合并,跳过 reserved 前缀(`report-` / `health-` / `_`)。所以一个 skill 可以把
12
10
  * sample 拆到多文件(`workflow.json` + `platform.json`),也可以单 `samples.json`,
13
11
  * 两种都行。
14
12
  *
15
- * 多 treatment 评测必须显式传 `--samples`(因为「找哪个 skill 的 bundled samples」
16
- * 没有唯一答案)。本函数仅处理单 treatment 与零 treatment 情况。
13
+ * 多 treatment 评测不会触发 skill-local 自动发现(因为「找哪个 skill 的 bundled samples」
14
+ * 没有唯一答案),只回到项目级 samples 兜底。
17
15
  */
18
- import { resolve, join, dirname } from 'node:path';
19
- import { existsSync, statSync } from 'node:fs';
16
+ import { findProjectSamplesFile, findSingleTreatmentSamplesPath } from '../../../inputs/sample-locator.js';
20
17
  export function discoverSamplesPath(values, skillDir) {
21
18
  const treatmentRaw = values.treatment;
22
19
  const treatments = treatmentRaw
23
20
  ? treatmentRaw.split(',').map((v) => v.trim()).filter(Boolean)
24
21
  : [];
25
22
  if (treatments.length === 1) {
26
- const expr = treatments[0];
27
- const resolved = resolve(expr);
28
- if (existsSync(resolved)) {
29
- const treatmentDir = statSync(resolved).isDirectory() ? resolved : dirname(resolved);
30
- const omkDir = join(treatmentDir, '.omk');
31
- if (existsSync(omkDir))
32
- return omkDir;
33
- for (const name of ['eval-samples.json', 'eval-samples.yaml', 'eval-samples.yml']) {
34
- if (existsSync(join(treatmentDir, name)))
35
- return join(treatmentDir, name);
36
- }
37
- }
38
- const omkDir = join(skillDir, expr, '.omk');
39
- if (existsSync(omkDir))
40
- return omkDir;
23
+ const samplesPath = findSingleTreatmentSamplesPath(treatments[0], skillDir, process.cwd());
24
+ if (samplesPath)
25
+ return samplesPath;
41
26
  }
42
- let cwdFile = 'eval-samples.json';
43
- if (!existsSync(resolve(cwdFile))) {
44
- if (existsSync(resolve('eval-samples.yaml')))
45
- cwdFile = 'eval-samples.yaml';
46
- else if (existsSync(resolve('eval-samples.yml')))
47
- cwdFile = 'eval-samples.yml';
48
- }
49
- return cwdFile;
27
+ return findProjectSamplesFile(process.cwd()) ?? 'eval-samples.json';
50
28
  }
@@ -41,10 +41,10 @@ export function parseRunConfig(values) {
41
41
  ? loadEvalConfig(values.config)
42
42
  : null;
43
43
  const skillDir = resolve(values['skill-dir'] ?? 'skills');
44
- // 2) Resolve samples path: CLI > config > <skillDir>/<treatment>/.omk/ discovery > cwd default.
45
- // The .omk/ discovery only fires when exactly one --treatment is given, so omk knows
46
- // which skill's bundled samples to use. The dir form (loadSamples handles both file + dir)
47
- // means a skill can split samples across multiple files (workflow.json / platform.json / ...).
44
+ // 2) Resolve samples path: CLI > config > single-treatment skill-local discovery > cwd project default.
45
+ // Skill-local discovery only fires when exactly one --treatment is given, so omk knows
46
+ // which skill's bundled samples to use. The `<skill>/.omk/` dir form (loadSamples handles
47
+ // both file + dir) means a skill can split samples across multiple files.
48
48
  const cliSamples = values.samples;
49
49
  let samplesFile;
50
50
  if (cliSamples) {
@@ -1,10 +1,11 @@
1
1
  import { resolve, join, dirname, basename } from 'node:path';
2
2
  import { existsSync, statSync } from 'node:fs';
3
3
  import { tCli } from './i18n.js';
4
+ import { findFlatSkillSamplesPath, findProjectSamplesFile, findSkillSamplesPath, skillLocalSamplesDir, } from '../../inputs/sample-locator.js';
4
5
  // 统一 skill 入参解析:既接受 SKILL.md 文件(老式 + flat skill),也接受 directory-skill
5
- // 目录(skills/foo/ 自动找 foo/SKILL.md)。samples 发现优先级:.omk/ 目录(loadSamples
6
- // 支持目录模式,自动 glob 多文件) > eval-samples.json > .yaml > .yml,fallback 是
7
- // .omk/ 目录路径(给上游 "samples 不存在" 的提示一个稳定路径)。
6
+ // 目录(skills/foo/ 自动找 foo/SKILL.md)。目录-skill 以 `<skill>/.omk/` 为标准
7
+ // samples 命名空间;扁平 .md 兼容 paired sidecar,找不到时回到项目级
8
+ // `eval-samples.json`。
8
9
  //
9
10
  // 错误用 tCli 走 i18n,调用方直接 console.error err.message 给用户看,zh/en 都要正确。
10
11
  export function resolveSkillInput(input, lang) {
@@ -26,16 +27,13 @@ export function resolveSkillInput(input, lang) {
26
27
  skillPath = resolved;
27
28
  skillDir = dirname(resolved);
28
29
  }
29
- const omkDir = join(skillDir, '.omk');
30
- const candidates = [
31
- ...(existsSync(omkDir) ? [omkDir] : []),
32
- join(skillDir, 'eval-samples.json'),
33
- join(skillDir, 'eval-samples.yaml'),
34
- join(skillDir, 'eval-samples.yml'),
35
- ];
36
- // fallback 到 .omk/ 目录:上游会报 "no sample files found in directory" 引导用户创建
37
- const samplesPath = candidates.find(existsSync) ?? omkDir;
38
30
  // 形态以解析后的 skillPath 命名为准:目录-skill 的 skillPath 总是 `.../SKILL.md`。
39
31
  const isDirectorySkill = basename(skillPath) === 'SKILL.md';
32
+ const flatSkillName = basename(skillPath).replace(/\.md$/i, '');
33
+ const samplesPath = isDirectorySkill
34
+ ? findSkillSamplesPath(skillDir) ?? skillLocalSamplesDir(skillDir)
35
+ : findFlatSkillSamplesPath(skillDir, flatSkillName)
36
+ ?? findProjectSamplesFile(process.cwd())
37
+ ?? 'eval-samples.json';
40
38
  return { skillPath, skillDir, samplesPath, isDirectorySkill };
41
39
  }
@@ -9,6 +9,7 @@
9
9
  */
10
10
  import { existsSync, statSync } from 'node:fs';
11
11
  import { basename, dirname, join, resolve } from 'node:path';
12
+ import { runFileSuffix } from '../eval-core/artifact-file-names.js';
12
13
  import { discoverVariants, resolveArtifacts } from '../inputs/skill-loader.js';
13
14
  import { DOCTOR_REPORT_SCHEMA_VERSION, isComposerRule } from '../types/doctor.js';
14
15
  import { getRegisteredRules } from './rules.js';
@@ -163,14 +164,8 @@ function inferSkillPath(artifact, baseDir) {
163
164
  // ---------------------------------------------------------------------------
164
165
  // Public entry
165
166
  // ---------------------------------------------------------------------------
166
- let reportIdCounter = 0;
167
167
  function nextReportId() {
168
- reportIdCounter += 1;
169
- const ts = new Date().toISOString().replace(/[-:.]/g, '').slice(0, 15);
170
- // 进程内计数器只防同进程同秒撞;跨进程同秒(两个 omk doctor 并发 / 不同项目)仍会撞同 id,
171
- // 经机器级卡片 dedup 当唯一键用时会静默并掉一份。加 4 位随机根治跨进程撞名(id 是标签非测量数)。
172
- const rand = Math.random().toString(36).slice(2, 6);
173
- return `doctor-${ts}-${reportIdCounter}-${rand}`;
168
+ return `doctor-${runFileSuffix()}`;
174
169
  }
175
170
  function readCliVersion() {
176
171
  // 不依赖 package.json import(避免 type 解析复杂度);用环境变量或退回 'unknown'
@@ -133,8 +133,8 @@ export const DOCTOR_MESSAGES = {
133
133
  en: 'directory-skill missing SKILL.md entry file',
134
134
  },
135
135
  'cli.doctor.skill_metadata.hint.frontmatter': {
136
- zh: 'front-matter 用 YAML 语法,key: value 或 - item 形式。可参考 examples/multi-skills 下的 skill 写法',
137
- en: 'front-matter uses YAML syntax (key: value or - item). See examples/multi-skills for reference',
136
+ zh: 'front-matter 用 YAML 语法,key: value 或 - item 形式。可参考 examples/skill-map-showcase/skills/release-readiness 的目录式 skill 写法',
137
+ en: 'front-matter uses YAML syntax (key: value or - item). See examples/skill-map-showcase/skills/release-readiness for a directory-skill reference',
138
138
  },
139
139
  'cli.doctor.skill_metadata.hint.hardrules': {
140
140
  zh: 'hardRules 必须写在 SKILL.md front-matter 中,格式为 hardRules: [{ id, rule, expectedBehavior }];id 要稳定且唯一,expectedBehavior 写可观察行为。',
@@ -0,0 +1,15 @@
1
+ export declare const REPORT_FILE_SUFFIX = ".report.json";
2
+ export declare const GRAPH_FILE_SUFFIX = ".graph.json";
3
+ export declare const CARD_FILE_SUFFIX = ".card.md";
4
+ export declare function safeArtifactFileStem(id: string): string;
5
+ export declare function reportFileName(stem: string): string;
6
+ export declare function reportFilePath(dir: string, stem: string): string;
7
+ export declare function reportFileStem(fileName: string): string | null;
8
+ export declare function isReportFileName(fileName: string): boolean;
9
+ export declare function graphFileName(stem: string): string;
10
+ export declare function cardFileName(stem: string): string;
11
+ export declare function runTimestamp(date?: Date): string;
12
+ export declare function randomRunToken(): string;
13
+ export declare function runFileSuffix(): string;
14
+ export declare function stripDomainPrefix(id: string, domain: string): string;
15
+ export declare function doctorReportFileStem(skillName: string, reportId: string): string;
@@ -0,0 +1,45 @@
1
+ import { join } from 'node:path';
2
+ export const REPORT_FILE_SUFFIX = '.report.json';
3
+ export const GRAPH_FILE_SUFFIX = '.graph.json';
4
+ export const CARD_FILE_SUFFIX = '.card.md';
5
+ export function safeArtifactFileStem(id) {
6
+ return id.replaceAll(/[/\\:*?"<>|]/g, '_');
7
+ }
8
+ export function reportFileName(stem) {
9
+ return `${safeArtifactFileStem(stem)}${REPORT_FILE_SUFFIX}`;
10
+ }
11
+ export function reportFilePath(dir, stem) {
12
+ return join(dir, reportFileName(stem));
13
+ }
14
+ export function reportFileStem(fileName) {
15
+ return fileName.endsWith(REPORT_FILE_SUFFIX)
16
+ ? fileName.slice(0, -REPORT_FILE_SUFFIX.length)
17
+ : null;
18
+ }
19
+ export function isReportFileName(fileName) {
20
+ return reportFileStem(fileName) !== null;
21
+ }
22
+ export function graphFileName(stem) {
23
+ return `${safeArtifactFileStem(stem)}${GRAPH_FILE_SUFFIX}`;
24
+ }
25
+ export function cardFileName(stem) {
26
+ return `${safeArtifactFileStem(stem)}${CARD_FILE_SUFFIX}`;
27
+ }
28
+ export function runTimestamp(date = new Date()) {
29
+ const pad = (n) => String(n).padStart(2, '0');
30
+ return `${date.getFullYear()}${pad(date.getMonth() + 1)}${pad(date.getDate())}T${pad(date.getHours())}${pad(date.getMinutes())}${pad(date.getSeconds())}`;
31
+ }
32
+ export function randomRunToken() {
33
+ return Math.random().toString(36).slice(2, 6);
34
+ }
35
+ export function runFileSuffix() {
36
+ return `${runTimestamp()}-${randomRunToken()}`;
37
+ }
38
+ export function stripDomainPrefix(id, domain) {
39
+ const safeId = safeArtifactFileStem(id);
40
+ const prefix = `${domain}-`;
41
+ return safeId.startsWith(prefix) ? safeId.slice(prefix.length) : safeId;
42
+ }
43
+ export function doctorReportFileStem(skillName, reportId) {
44
+ return `${safeArtifactFileStem(skillName)}-${stripDomainPrefix(reportId, 'doctor')}`;
45
+ }
@@ -33,7 +33,7 @@ export interface PersistableReport {
33
33
  }
34
34
  export declare function persistReport(report: PersistableReport, outputDir: string | null): string | null;
35
35
  /**
36
- * run id 的时间戳后缀 `YYYYMMDD-HHmmss-rand4`。
36
+ * run id 的时间戳后缀 `YYYYMMDDTHHmmss-rand4`。
37
37
  * 含秒 + 4 位随机:id 是 run 标签(非测量数),但被 studio 机器级 dedup 与 managed 证据 (reportId,
38
38
  * contentHash) 去重当唯一键用。分钟级会让跨项目 / 同分钟重跑撞同 id → 索引静默顶掉一份、managed 错并一条。
39
39
  * 秒+随机根治撞名,保证每次 run 全局唯一。供 generateRunId 与 evolve 合并 id 共用,避免靠 split 反解格式。
@@ -5,6 +5,8 @@ import { createHash } from 'node:crypto';
5
5
  import { fileURLToPath } from 'node:url';
6
6
  import { DEFAULT_REPORTS_DIR } from './default-dirs.js';
7
7
  import { indexReportWrite } from './artifact-index.js';
8
+ import { randomRunToken, reportFilePath, runTimestamp } from './artifact-file-names.js';
9
+ import { persistEvalGraphSidecar } from '../artifact-graph/eval.js';
8
10
  import { buildVariantSummary } from './schema.js';
9
11
  import { buildVariantConfig, resolveExecutionStrategy } from './execution-strategy.js';
10
12
  import { getJudgePromptHash } from '../grading/judge.js';
@@ -275,35 +277,53 @@ export function aggregateReport({ runId, variants, model, judgeModel, noJudge, e
275
277
  }])),
276
278
  };
277
279
  }
280
+ function isEvaluationReport(report) {
281
+ return report['kind'] === 'evaluation';
282
+ }
283
+ function persistEvalGraphSidecarSafely(report, outputDir, sourcePath) {
284
+ if (!isEvaluationReport(report))
285
+ return;
286
+ try {
287
+ persistEvalGraphSidecar({ report, outputDir, sourcePath, fileStem: report.id });
288
+ }
289
+ catch (err) {
290
+ const message = err instanceof Error ? err.message : String(err);
291
+ process.stderr.write(`[omk] 写入 eval 图谱失败:${message}\n`);
292
+ }
293
+ }
278
294
  export function persistReport(report, outputDir) {
279
295
  if (!outputDir)
280
296
  return null;
281
297
  if (!existsSync(outputDir))
282
298
  mkdirSync(outputDir, { recursive: true });
283
- const filePath = join(outputDir, `${report.id}.json`);
299
+ const filePath = reportFilePath(outputDir, report.id);
284
300
  writeFileSync(filePath, JSON.stringify(report, null, 2));
301
+ persistEvalGraphSidecarSafely(report, outputDir, filePath);
285
302
  // 产物发现索引:报告落项目本地后,best-effort 追加全局轻卡片,让 omk studio 跨项目聚合成机器级总览。
286
303
  // 永不抛、永不阻断报告落盘(正文是 source of truth)。
287
304
  indexReportWrite(report, filePath, outputDir);
288
305
  return filePath;
289
306
  }
290
307
  /**
291
- * run id 的时间戳后缀 `YYYYMMDD-HHmmss-rand4`。
308
+ * run id 的时间戳后缀 `YYYYMMDDTHHmmss-rand4`。
292
309
  * 含秒 + 4 位随机:id 是 run 标签(非测量数),但被 studio 机器级 dedup 与 managed 证据 (reportId,
293
310
  * contentHash) 去重当唯一键用。分钟级会让跨项目 / 同分钟重跑撞同 id → 索引静默顶掉一份、managed 错并一条。
294
311
  * 秒+随机根治撞名,保证每次 run 全局唯一。供 generateRunId 与 evolve 合并 id 共用,避免靠 split 反解格式。
295
312
  */
296
313
  export function runIdSuffix() {
297
- const d = new Date();
298
- const pad = (n) => String(n).padStart(2, '0');
299
- const date = `${d.getFullYear()}${pad(d.getMonth() + 1)}${pad(d.getDate())}`;
300
- const time = `${pad(d.getHours())}${pad(d.getMinutes())}${pad(d.getSeconds())}`;
301
- const rand = Math.random().toString(36).slice(2, 6);
302
- return `${date}-${time}-${rand}`;
314
+ return `${runTimestamp()}-${randomRunToken()}`;
315
+ }
316
+ function safeRunSubject(subject) {
317
+ const sanitized = subject
318
+ .replaceAll(/[\\/:]/g, '-')
319
+ .replaceAll(/[^a-zA-Z0-9._@-]/g, '_')
320
+ .replace(/^-+|-+$/g, '');
321
+ return sanitized || 'run';
322
+ }
323
+ function primaryRunSubject(variants) {
324
+ const nonBaseline = variants.filter((variant) => variant !== 'baseline');
325
+ return nonBaseline.at(-1) ?? variants.at(-1) ?? 'run';
303
326
  }
304
327
  export function generateRunId(variants) {
305
- const variantPart = variants
306
- .map((variant) => variant.replaceAll(/[\\/:]/g, '-').replaceAll(/[^a-zA-Z0-9._@-]/g, '_'))
307
- .join('-vs-');
308
- return `${variantPart}-${runIdSuffix()}`;
328
+ return `${safeRunSubject(primaryRunSubject(variants))}-${runIdSuffix()}`;
309
329
  }
@@ -1,14 +1,16 @@
1
1
  import { existsSync, readdirSync } from 'node:fs';
2
2
  import { join } from 'node:path';
3
3
  import { DEFAULT_OBSERVE_HEALTH_DIR, DEFAULT_DOCTORS_DIR, DEFAULT_REPORTS_DIR } from './default-dirs.js';
4
+ import { isReportFileName } from './artifact-file-names.js';
5
+ import { migrateLegacyReportFiles } from './report-file-migration.js';
4
6
  /**
5
7
  * 测量产物的「项目优先 → 全局兜底」目录解析,镜像 managed 的 `resolveManagedDir`
6
8
  * (src/managed/store.ts)。测量产物绑用例集上下文(construct validity,不可全局化),
7
9
  * 默认落项目 `.omk/`,全局作显式 opt-in。
8
10
  *
9
- * 「记录优先」—— 目录里有匹配 `.json` 才算数(不是「目录存在」),与 managed 同口径,
10
- * 避免空项目目录遮蔽全局数据。项目目录按**调用时** `cwd()` 求值(函数,不是 import 时
11
- * 冻结的常量),studio 长会话 per-request 解析才正确。
11
+ * 「记录优先」—— 目录里有匹配 report 文件才算数(不是「目录存在」),与 managed 同口径,
12
+ * 避免空项目目录遮蔽全局数据。项目目录按**调用时** `cwd()` 求值(函数,不是 import 时
13
+ * 冻结的常量),studio 长会话 per-request 解析才正确。
12
14
  */
13
15
  /** dir 里是否有至少一个满足 match 的文件(短路,不读文件内容)。 */
14
16
  function hasMatchingFile(dir, match) {
@@ -22,7 +24,7 @@ function hasMatchingFile(dir, match) {
22
24
  }
23
25
  }
24
26
  // —— observe-health(skill 健康度报告)——
25
- // 文件名 `{timestamp}-observe-health.json`,匹配后缀避免别的 .json 误翻转 resolver。
27
+ // 文件名 `{run}.report.json`,目录已表达 observe-health 域。
26
28
  /** 项目级 observe-health 目录(相对调用时 cwd)。 */
27
29
  export function projectObserveHealthDir(cwd = process.cwd()) {
28
30
  return join(cwd, '.omk', 'observe-health');
@@ -34,7 +36,9 @@ export function globalObserveHealthDir() {
34
36
  /** 权威 observe-health 目录:项目有报告取项目,否则全局有取全局,都空回项目(同 resolveManagedDir)。
35
37
  * `global` 可注入(默认真实全局目录),仅供测试用受控 temp 目录复现 project↔global 兜底。 */
36
38
  export function resolveObserveHealthDir(dir = projectObserveHealthDir(), global = globalObserveHealthDir()) {
37
- const has = (d) => hasMatchingFile(d, (f) => f.endsWith('-observe-health.json'));
39
+ migrateLegacyReportFiles(dir, 'observe-health');
40
+ migrateLegacyReportFiles(global, 'observe-health');
41
+ const has = (d) => hasMatchingFile(d, isReportFileName);
38
42
  if (has(dir))
39
43
  return dir;
40
44
  if (dir !== global && has(global))
@@ -42,7 +46,7 @@ export function resolveObserveHealthDir(dir = projectObserveHealthDir(), global
42
46
  return dir;
43
47
  }
44
48
  // —— doctors(体检报告)——
45
- // 文件名 `{skill}-{id}.json`,谓词用「存在任意 .json」即可(真解析在下游 scanDoctorReports)。
49
+ // 文件名 `{skill}-{run}.report.json`,谓词用 report 后缀即可(真解析在下游 scanDoctorReports)。
46
50
  /** 项目级 doctors 目录(相对调用时 cwd)。 */
47
51
  export function projectDoctorsDir(cwd = process.cwd()) {
48
52
  return join(cwd, '.omk', 'doctors');
@@ -54,7 +58,9 @@ export function globalDoctorsDir() {
54
58
  /** 权威 doctors 目录:项目有报告取项目,否则全局有取全局,都空回项目(同 resolveManagedDir)。
55
59
  * `global` 可注入(默认真实全局目录),仅供测试用受控 temp 目录复现 project↔global 兜底。 */
56
60
  export function resolveDoctorsDir(dir = projectDoctorsDir(), global = globalDoctorsDir()) {
57
- const has = (d) => hasMatchingFile(d, (f) => f.endsWith('.json'));
61
+ migrateLegacyReportFiles(dir, 'doctor');
62
+ migrateLegacyReportFiles(global, 'doctor');
63
+ const has = (d) => hasMatchingFile(d, isReportFileName);
58
64
  if (has(dir))
59
65
  return dir;
60
66
  if (dir !== global && has(global))
@@ -0,0 +1,10 @@
1
+ export type LegacyReportDomain = 'report' | 'doctor' | 'observe-health' | 'observe-inbox';
2
+ /**
3
+ * One-shot migration for pre-`.report.json` run artifacts.
4
+ *
5
+ * This is deliberately scoped to measurement report directories. It does not make
6
+ * bare `.json` a normal reader format again; it just renames known legacy report
7
+ * files into the canonical name so old local data can be discovered, rotated, and
8
+ * eventually garbage-collected by the new code paths.
9
+ */
10
+ export declare function migrateLegacyReportFiles(dir: string, domain: LegacyReportDomain): void;