oh-my-knowledge 0.41.0 → 0.43.0

package/dist/cli/commands/sample.js

@@ -9,6 +9,7 @@ import { CliExit } from '../lib/cli-exit.js';
 import { tCli } from '../lib/i18n.js';
 import { projectReportsDir, globalReportsDir } from '../../eval-core/measurement-dirs.js';
 import { loadSamples, parseYaml, listSampleFilesInDir } from '../../inputs/load-samples.js';
+import { defaultFlatSkillSamplesFile, defaultSkillLocalSamplesFile, findFlatSkillSamplesPath, findSkillSamplesPath, } from '../../inputs/sample-locator.js';
 import { hashSample } from '../../eval-core/evaluation-reporting.js';
 import { hashArtifactSource } from '../../inputs/content-hash.js';
 function isRecord(value) {
@@ -233,23 +234,23 @@ async function runSampleFix(args, flags, lang) {
         console.error(lang === 'zh' ? '请指定 skill 路径，如: omk sample skills/my-skill/SKILL.md --fix' : 'Specify skill path: omk sample skills/my-skill/SKILL.md --fix');
         throw new CliExit(1);
     }
-    const resolvedSkillPath = resolve(skillPath);
-    if (!existsSync(resolvedSkillPath)) {
-        console.error(lang === 'zh' ? `skill 文件不存在: ${resolvedSkillPath}` : `Skill file not found: ${resolvedSkillPath}`);
+    const { resolveSkillInput } = await import('../lib/resolve-skill-input.js');
+    let resolvedInput;
+    try {
+        resolvedInput = resolveSkillInput(skillPath, lang);
+    }
+    catch (err) {
+        console.error(err instanceof Error ? err.message : String(err));
         throw new CliExit(1);
     }
-    const isDir = basename(resolvedSkillPath) === 'SKILL.md';
-    const skillDir = isDir ? dirname(resolvedSkillPath) : dirname(resolvedSkillPath);
-    const samplesInput = isDir
-        ? join(skillDir, '.omk')
-        : resolve('eval-samples.json');
+    const samplesInput = resolvedInput.samplesPath;
     if (!existsSync(samplesInput)) {
         console.error(lang === 'zh' ? `samples 路径不存在: ${samplesInput}，先运行 omk sample 生成` : `Samples path not found: ${samplesInput}, run omk sample first`);
         throw new CliExit(1);
     }
-    const defaultTreatmentName = isDir
-        ? basename(skillDir)
-        : basename(resolvedSkillPath, extname(resolvedSkillPath));
+    const defaultTreatmentName = resolvedInput.isDirectorySkill
+        ? basename(resolvedInput.skillDir)
+        : basename(resolvedInput.skillPath, extname(resolvedInput.skillPath));
     const treatmentName = flags.treatment ?? defaultTreatmentName;
     process.stderr.write(lang === 'zh' ? `🔍 正在查找 ${treatmentName} 的最新评测报告...\n` : `🔍 Scanning latest report for ${treatmentName}...\n`);
     // 显式 --reports-dir 固定该目录;默认 overlay(项目 .omk/reports 盖全局),findByVariant 记录优先看项目、
@@ -277,7 +278,7 @@ async function runSampleFix(args, flags, lang) {
         throw new CliExit(1);
     }
     const samples = loadedSamples.samples;
-    const skillContent = readFileSync(resolvedSkillPath, 'utf-8');
+    const skillContent = readFileSync(resolvedInput.skillPath, 'utf-8');
     const sampleDesignIds = collectSampleDesignFailureIds(report, treatmentName);
     const sampleDesignCount = sampleDesignIds.size;
     if (sampleDesignCount === 0) {
@@ -286,7 +287,7 @@ async function runSampleFix(args, flags, lang) {
     }
     // 当前内容指纹走整树哈,与 eval 报告口径一致:dir-skill(用户传 .../SKILL.md)哈整棵 skill 目录、
     // 单文件 .md 哈单文件字节。
-    const currentContentHash = hashArtifactSource(isDir ? skillDir : resolvedSkillPath, isDir);
+    const currentContentHash = hashArtifactSource(resolvedInput.isDirectorySkill ? resolvedInput.skillDir : resolvedInput.skillPath, resolvedInput.isDirectorySkill);
     try {
         assertFixReportMatchesCurrentInputs({
             report,
@@ -435,24 +436,29 @@ async function runSample(args, flags, lang) {
             let name;
             let skillPath;
             let samplesPath;
+            let existingSamplesPath;
             const fullPath = join(skillDir, entry);
             if (entry.endsWith('.md') && !entry.endsWith('.eval-samples.json')) {
                 name = entry.slice(0, -3);
                 skillPath = fullPath;
-                samplesPath = join(skillDir, `${name}.eval-samples.json`);
+                samplesPath = defaultFlatSkillSamplesFile(skillDir, name);
+                existingSamplesPath = findFlatSkillSamplesPath(skillDir, name);
             }
             else if (statSync(fullPath).isDirectory()) {
                 const skillMd = join(fullPath, 'SKILL.md');
                 if (!existsSync(skillMd))
                     continue;
+                if (existsSync(join(skillDir, `${entry}.md`)))
+                    continue;
                 name = entry;
                 skillPath = skillMd;
-                samplesPath = join(fullPath, '.omk', 'samples.json');
+                samplesPath = defaultSkillLocalSamplesFile(fullPath);
+                existingSamplesPath = findSkillSamplesPath(fullPath);
             }
             else {
                 continue;
             }
-            if (existsSync(samplesPath)) {
+            if (existingSamplesPath) {
                 process.stderr.write(tCli('cli.gen.skill_skipped_existing', lang, { name }));
                 continue;
             }

package/dist/cli/lib/i18n-dict/common.d.ts

@@ -1,3 +1,3 @@
 import type { CliMessage } from './types.js';
-export type CommonMessageKey = 'cli.common.unknown_domain' | 'cli.common.error_prefix' | 'cli.common.skill_dir_not_found' | 'cli.common.skill_file_not_found' | 'cli.common.skill_dir_no_skill_md' | 'cli.common.report_not_found' | 'cli.common.no_judge_model' | 'cli.common.judge_models_single_only' | 'cli.common.warn_load_samples_failed' | 'cli.update.new_version_available' | 'cli.update.box_title' | 'cli.update.box_version_line' | 'cli.update.box_upgrade_line' | 'cli.update.box_silence_line' | 'cli.observe.view_hint' | 'cli.observe.observation_recorded' | 'cli.observe.production_gap' | 'cli.studio.started' | 'cli.studio.stop_hint' | 'cli.studio.open_failed' | 'cli.doctor.no_skill_found' | 'cli.doctor.samples_detected' | 'cli.doctor.progress_skill_start' | 'cli.doctor.progress_skill_done';
+export type CommonMessageKey = 'cli.common.unknown_domain' | 'cli.common.error_prefix' | 'cli.common.skill_dir_not_found' | 'cli.common.skill_file_not_found' | 'cli.common.skill_dir_no_skill_md' | 'cli.common.report_not_found' | 'cli.common.no_judge_model' | 'cli.common.judge_models_single_only' | 'cli.common.warn_load_samples_failed' | 'cli.common.deprecated_skill_samples_path' | 'cli.common.samples_not_found' | 'cli.update.new_version_available' | 'cli.update.box_title' | 'cli.update.box_version_line' | 'cli.update.box_upgrade_line' | 'cli.update.box_silence_line' | 'cli.observe.view_hint' | 'cli.observe.observation_recorded' | 'cli.observe.production_gap' | 'cli.studio.started' | 'cli.studio.stop_hint' | 'cli.studio.open_failed' | 'cli.doctor.no_skill_found' | 'cli.doctor.samples_detected' | 'cli.doctor.progress_skill_start' | 'cli.doctor.progress_skill_done';
 export declare const commonDict: Record<CommonMessageKey, CliMessage>;

package/dist/cli/lib/i18n-dict/common.js

@@ -35,6 +35,14 @@ export const commonDict = {
         zh: '⚠ 加载 samples 文件失败 ({path}): {message}\n',
         en: '⚠ Failed to load samples file ({path}): {message}\n',
     },
+    'cli.common.deprecated_skill_samples_path': {
+        zh: '⚠ 发现旧的目录 skill 用例位置：{oldPath}。目录 skill 的私有用例已改放到 {newPath}，请迁移。\n',
+        en: '⚠ Found deprecated directory skill samples path: {oldPath}. Directory skill samples now live at {newPath}; please migrate.\n',
+    },
+    'cli.common.samples_not_found': {
+        zh: '未找到评测用例：{path}。请通过 --samples 指定文件，或创建项目级 eval-samples.json；单 treatment 目录 skill 请使用 <skill>/.omk/samples.json。',
+        en: 'Eval samples not found: {path}. Pass --samples, create project-level eval-samples.json, or use <skill>/.omk/samples.json for a single-treatment directory skill.',
+    },
     'cli.update.new_version_available': {
         zh: '\n💡 新版本可用：{old} → {new}，运行 npm i -g oh-my-knowledge@latest 升级\n\n',
         en: '\n💡 New version available: {old} → {new}, run npm i -g oh-my-knowledge@latest to upgrade\n\n',

package/dist/cli/lib/i18n-dict/help.js

@@ -170,20 +170,21 @@ omk sample——生成或补齐 eval-samples 评测用例
   omk sample --batch [--skill-dir <dir>] [options]
 
 输出位置（默认）：
-  <skill>/SKILL.md  → <skill>/.omk/samples.json（omk 标准约定）
-  其他 .md 路径    → 当前目录的 eval-samples.json（兜底）
+  目录 skill（<skill>/SKILL.md） → <skill>/.omk/samples.json（omk 标准约定）
+  扁平 .md（单次）               → 当前目录的 eval-samples.json（项目级兜底）
+  扁平 .md（--batch）            → <skill-dir>/<name>.eval-samples.json（兼容 paired 布局）
 
 选项：
   --count <n>                         强制生成 N 条（不指定时由 LLM 按 skill 类型自动判断：
                                        工作流型 6-8 条 / 原子型 4-6 条 / 混合型 5-7 条）
   --model <name>                      生成模型（默认：opus；lean+effort-low 已自动开,想省钱可改 sonnet/haiku）
   --focus <text>                      自然语言指定希望覆盖的场景（追加到 prompt，优先级高于自由发挥）
-  --batch                             为 skill 目录下缺少 eval-samples 的 skill 批量生成
+  --batch                             为 skill 目录下缺少 samples 的 skill 批量生成
   --skill-dir <path>                  skill 目录（batch 使用，默认：skills）
 
 示例：
-  omk improve samples skills/req-tool.md
-  omk improve samples skills/req-tool.md --count 8 \\
+  omk sample skills/req-tool.md
+  omk sample skills/req-tool.md --count 8 \\
     --focus "重点覆盖 tag 查询走 PROJECT 空 → WORKSPACE 兜底的多步流程，以及 search 失败的错误路径"
 `,
         en: `
@@ -194,20 +195,21 @@ Usage:
   omk sample --batch [--skill-dir <dir>] [options]
 
 Output path (default):
-  <skill>/SKILL.md  → <skill>/.omk/samples.json (omk standard layout)
-  other .md paths   → ./eval-samples.json in current directory (fallback)
+  directory skill (<skill>/SKILL.md) → <skill>/.omk/samples.json (omk standard layout)
+  flat .md (single)                  → ./eval-samples.json in current directory (project fallback)
+  flat .md (--batch)                 → <skill-dir>/<name>.eval-samples.json (compatible paired layout)
 
 Options:
   --count <n>                         Force N samples (omit to let LLM auto-decide by skill type:
                                        workflow 6-8 / atomic 4-6 / mixed 5-7)
   --model <name>                      Generation model (default: opus; lean+effort-low applied; pass --model sonnet/haiku to save cost)
   --focus <text>                      Natural-language scenario hints appended to the prompt (overrides freeform diversity)
-  --batch                             Generate for skills that are missing eval-samples
+  --batch                             Generate for skills that are missing samples
   --skill-dir <path>                  Skill directory for batch mode (default: skills)
 
 Examples:
-  omk improve samples skills/req-tool.md
-  omk improve samples skills/req-tool.md --count 8 \\
+  omk sample skills/req-tool.md
+  omk sample skills/req-tool.md --count 8 \\
     --focus "Cover PROJECT-empty → WORKSPACE-fallback multi-step tag lookup and the search-failure error path"
 `,
     },

package/dist/cli/lib/i18n-dict/init.d.ts

@@ -1,3 +1,3 @@
 import type { CliMessage } from './types.js';
-export type InitMessageKey = 'cli.init.scaffolded' | 'cli.init.next_steps_title' | 'cli.init.next_step_edit_samples' | 'cli.init.next_step_edit_skills' | 'cli.init.next_step_run' | 'cli.init.note_codex_executor';
+export type InitMessageKey = 'cli.init.scaffolded' | 'cli.init.next_steps_title' | 'cli.init.next_step_run' | 'cli.init.next_step_executor' | 'cli.init.next_step_customize' | 'cli.init.note_codex_executor';
 export declare const initDict: Record<InitMessageKey, CliMessage>;

package/dist/cli/lib/i18n-dict/init.js

@@ -1,23 +1,26 @@
 export const initDict = {
     'cli.init.scaffolded': {
-        zh: '已初始化 omk 项目: {dir}',
+        zh: '已初始化 omk 项目：{dir}',
         en: 'omk project initialized at: {dir}',
     },
     'cli.init.next_steps_title': {
-        zh: '下一步:',
+        zh: '下一步：',
         en: 'Next steps:',
     },
-    'cli.init.next_step_edit_samples': {
-        zh: '  1. 编辑 eval-samples.json，加入你要测的评测用例',
-        en: '  1. Edit eval-samples.json to add your test cases',
+    // 先让用户「无需改任何文件直接跑通」——脚手架的用例与 skill 本身可跑(已过合规校验),
+    // 跑出第一份报告是冷启动最该先发生的事;「换成你自己的」放到跑通之后。这也消除了
+    // 主 README「不用改任何文件」与旧 init「先编辑」的矛盾。
+    'cli.init.next_step_run': {
+        zh: '  1. 直接跑通（无需先改任何文件）：omk eval --control code-review-v1 --treatment code-review-v2',
+        en: '  1. Run it as-is (no edits needed): omk eval --control code-review-v1 --treatment code-review-v2',
     },
-    'cli.init.next_step_edit_skills': {
-        zh: '  2. 编辑 skills/code-review-v1/SKILL.md 和 skills/code-review-v2/SKILL.md, 为两个 skill 版本填入实际内容',
-        en: '  2. Edit skills/code-review-v1/SKILL.md and skills/code-review-v2/SKILL.md with your skill versions',
+    'cli.init.next_step_executor': {
+        zh: '     默认执行器与评委用 claude CLI，需先装好并登录；想换别的模型或离线跑（无需 API key）见文档「执行器」。',
+        en: '     The default executor and judge use the claude CLI (install and log in first); to use another model or run offline (no API key) see the Executors docs.',
     },
-    'cli.init.next_step_run': {
-        zh: '  3. 运行: omk eval --control code-review-v1 --treatment code-review-v2',
-        en: '  3. Run: omk eval --control code-review-v1 --treatment code-review-v2',
+    'cli.init.next_step_customize': {
+        zh: '  2. 跑通后，把 skills/code-review-v1/SKILL.md 和 skills/code-review-v2/SKILL.md 与 eval-samples.json 换成你自己的 skill 和用例',
+        en: '  2. Once it runs, replace skills/code-review-v1/SKILL.md and skills/code-review-v2/SKILL.md and eval-samples.json with your own skills and cases',
     },
     'cli.init.note_codex_executor': {
         zh: '\n注: omk 评测时把 SKILL.md 整文(含 frontmatter)作为 system prompt 注入——跨 executor 一致(claude / codex / openai-api / gemini 都走同一条路径,不依赖任何 executor 的 native skill auto-discovery 或 Skill 工具机制)。frontmatter 在 prompt 头部对 model 行为无显著影响。\n模板带 Claude Code 兼容的 frontmatter(name + description)是为了让同一份 directory-skill 也能 deploy 到 Claude Code:把整个目录复制到 ~/.claude/skills/code-review-v1/(整目录,不是单个 SKILL.md),Claude SDK 才能识别。这是 omk 评测之外的 bonus,一份文件双向 dogfood。',

package/dist/cli/lib/i18n-dict/run.d.ts

@@ -1,3 +1,3 @@
 import type { CliMessage } from './types.js';
-export type RunMessageKey = 'cli.progress.preflight_starting' | 'cli.progress.sample_retry' | 'cli.progress.sample_error' | 'cli.progress.sample_executing' | 'cli.progress.sample_exec_done' | 'cli.progress.output_preview' | 'cli.progress.judging' | 'cli.progress.judged' | 'cli.progress.skipped' | 'cli.progress.sample_done' | 'cli.progress.sample_failed_done' | 'cli.run.invalid_repeat' | 'cli.run.invalid_judge_repeat' | 'cli.run.no_debias_length_active' | 'cli.run.invalid_bootstrap_samples' | 'cli.run.bootstrap_samples_too_large' | 'cli.run.dry_run_no_scores' | 'cli.run.skill_section' | 'cli.run.run_section' | 'cli.run.batch_complete' | 'cli.run.batch_verdict_header' | 'cli.run.batch_child_report_missing' | 'cli.run.eval_complete' | 'cli.run.tally' | 'cli.run.report_saved' | 'cli.run.evidence_recorded' | 'cli.run.evidence_recorded_unbound' | 'cli.run.report_only_gate_skipped' | 'cli.run.report_server_running' | 'cli.run.report_server_view' | 'cli.run.report_server_stop' | 'cli.run.no_serve_in_non_tty' | 'cli.run.no_serve_view_hint' | 'cli.run.gold_load_failed' | 'cli.run.gold_load_issue' | 'cli.run.contamination_warning' | 'cli.run.skip_connectivity_warning';
+export type RunMessageKey = 'cli.progress.preflight_starting' | 'cli.progress.sample_retry' | 'cli.progress.sample_error' | 'cli.progress.sample_executing' | 'cli.progress.sample_exec_done' | 'cli.progress.output_preview' | 'cli.progress.judging' | 'cli.progress.judged' | 'cli.progress.skipped' | 'cli.progress.sample_done' | 'cli.progress.sample_failed_done' | 'cli.run.invalid_repeat' | 'cli.run.invalid_holdout_ratio' | 'cli.run.invalid_judge_repeat' | 'cli.run.no_debias_length_active' | 'cli.run.invalid_bootstrap_samples' | 'cli.run.bootstrap_samples_too_large' | 'cli.run.dry_run_no_scores' | 'cli.run.skill_section' | 'cli.run.run_section' | 'cli.run.batch_complete' | 'cli.run.batch_verdict_header' | 'cli.run.batch_child_report_missing' | 'cli.run.eval_complete' | 'cli.run.tally' | 'cli.run.report_saved' | 'cli.run.evidence_recorded' | 'cli.run.evidence_recorded_unbound' | 'cli.run.report_only_gate_skipped' | 'cli.run.report_server_running' | 'cli.run.report_server_view' | 'cli.run.report_server_stop' | 'cli.run.no_serve_in_non_tty' | 'cli.run.no_serve_view_hint' | 'cli.run.gold_load_failed' | 'cli.run.gold_load_issue' | 'cli.run.contamination_warning' | 'cli.run.skip_connectivity_warning';
 export declare const runDict: Record<RunMessageKey, CliMessage>;

package/dist/cli/lib/i18n-dict/run.js

@@ -51,6 +51,10 @@ export const runDict = {
         zh: '⚠ --judge-repeat "{value}" 无效 (期望 ≥ 1 的整数), 已按 1 次 judge 执行\n',
         en: '⚠ --judge-repeat "{value}" is invalid (expected an integer ≥ 1), falling back to 1 judge call\n',
     },
+    'cli.run.invalid_holdout_ratio': {
+        zh: '⚠ --holdout-ratio "{value}" 无效 (期望 0 到 1 之间的小数), 已忽略、不做 holdout 切分\n',
+        en: '⚠ --holdout-ratio "{value}" is invalid (expected a fraction in (0, 1)), ignored — no holdout split\n',
+    },
     'cli.run.no_debias_length_active': {
         zh: 'ℹ --no-debias-length 已生效：judge prompt 去掉长度去偏指令（debias-off 变体），hash 与默认开启时不同。\n',
         en: 'ℹ --no-debias-length is active: the judge prompt drops the length-debias instruction (debias-off variant); its hash differs from the default.\n',

package/dist/cli/lib/parse-run-config/samples-discovery.d.ts

@@ -1,18 +1,16 @@
 /**
  * 当 CLI 未传 `--samples` 时,自动发现 sample 路径。
  *
- * 发现顺序(单 treatment 模式):
- *   1. treatment 是绝对 / 相对路径(文件 / 目录) → 找它所在目录的 `.omk/`
- *   2. fallback 找目录里的 `eval-samples.{json,yaml,yml}`
- *   3. fallback `<skillDir>/<treatmentName>/.omk/`
- *   4. 终极兜底 cwd 下的 `eval-samples.{json,yaml,yml}`
+ * 发现顺序:
+ *   1. 单 treatment → 找该 skill 私有 samples(`<skill>/.omk/`)或扁平 skill paired 文件
+ *   2. 终极兜底 cwd 下的项目级 `eval-samples.{json,yaml,yml}`
  *
  * `loadSamples` 在下游自己处理「文件 vs 目录」—— 目录模式 glob `*.{json,yaml,yml}`
  * 合并,跳过 reserved 前缀(`report-` / `health-` / `_`)。所以一个 skill 可以把
  * sample 拆到多文件(`workflow.json` + `platform.json`),也可以单 `samples.json`,
  * 两种都行。
  *
- * 多 treatment 评测必须显式传 `--samples`(因为「找哪个 skill 的 bundled samples」
- * 没有唯一答案)。本函数仅处理单 treatment 与零 treatment 情况。
+ * 多 treatment 评测不会触发 skill-local 自动发现(因为「找哪个 skill 的 bundled samples」
+ * 没有唯一答案),只回到项目级 samples 兜底。
  */
 export declare function discoverSamplesPath(values: Record<string, unknown>, skillDir: string): string;

package/dist/cli/lib/parse-run-config/samples-discovery.js

@@ -1,50 +1,28 @@
 /**
  * 当 CLI 未传 `--samples` 时,自动发现 sample 路径。
  *
- * 发现顺序(单 treatment 模式):
- *   1. treatment 是绝对 / 相对路径(文件 / 目录) → 找它所在目录的 `.omk/`
- *   2. fallback 找目录里的 `eval-samples.{json,yaml,yml}`
- *   3. fallback `<skillDir>/<treatmentName>/.omk/`
- *   4. 终极兜底 cwd 下的 `eval-samples.{json,yaml,yml}`
+ * 发现顺序:
+ *   1. 单 treatment → 找该 skill 私有 samples(`<skill>/.omk/`)或扁平 skill paired 文件
+ *   2. 终极兜底 cwd 下的项目级 `eval-samples.{json,yaml,yml}`
  *
  * `loadSamples` 在下游自己处理「文件 vs 目录」—— 目录模式 glob `*.{json,yaml,yml}`
  * 合并,跳过 reserved 前缀(`report-` / `health-` / `_`)。所以一个 skill 可以把
  * sample 拆到多文件(`workflow.json` + `platform.json`),也可以单 `samples.json`,
  * 两种都行。
  *
- * 多 treatment 评测必须显式传 `--samples`(因为「找哪个 skill 的 bundled samples」
- * 没有唯一答案)。本函数仅处理单 treatment 与零 treatment 情况。
+ * 多 treatment 评测不会触发 skill-local 自动发现(因为「找哪个 skill 的 bundled samples」
+ * 没有唯一答案),只回到项目级 samples 兜底。
  */
-import { resolve, join, dirname } from 'node:path';
-import { existsSync, statSync } from 'node:fs';
+import { findProjectSamplesFile, findSingleTreatmentSamplesPath } from '../../../inputs/sample-locator.js';
 export function discoverSamplesPath(values, skillDir) {
     const treatmentRaw = values.treatment;
     const treatments = treatmentRaw
         ? treatmentRaw.split(',').map((v) => v.trim()).filter(Boolean)
         : [];
     if (treatments.length === 1) {
-        const expr = treatments[0];
-        const resolved = resolve(expr);
-        if (existsSync(resolved)) {
-            const treatmentDir = statSync(resolved).isDirectory() ? resolved : dirname(resolved);
-            const omkDir = join(treatmentDir, '.omk');
-            if (existsSync(omkDir))
-                return omkDir;
-            for (const name of ['eval-samples.json', 'eval-samples.yaml', 'eval-samples.yml']) {
-                if (existsSync(join(treatmentDir, name)))
-                    return join(treatmentDir, name);
-            }
-        }
-        const omkDir = join(skillDir, expr, '.omk');
-        if (existsSync(omkDir))
-            return omkDir;
+        const samplesPath = findSingleTreatmentSamplesPath(treatments[0], skillDir, process.cwd());
+        if (samplesPath)
+            return samplesPath;
     }
-    let cwdFile = 'eval-samples.json';
-    if (!existsSync(resolve(cwdFile))) {
-        if (existsSync(resolve('eval-samples.yaml')))
-            cwdFile = 'eval-samples.yaml';
-        else if (existsSync(resolve('eval-samples.yml')))
-            cwdFile = 'eval-samples.yml';
-    }
-    return cwdFile;
+    return findProjectSamplesFile(process.cwd()) ?? 'eval-samples.json';
 }

package/dist/cli/lib/parse-run-config.d.ts

@@ -45,6 +45,9 @@ export interface RunConfig {
     retry?: number;
     resume?: string;
     layeredStats?: boolean;
+    /** --holdout-ratio R (0 < R < 1). Hold out a deterministic sample slice; report-finalize
+     *  computes train vs holdout composite (report.analysis.holdout) for the overfitting gate. */
+    holdoutRatio?: number;
     /** --judge-repeat N. Calls LLM judge N times per (sample × dimension). Default 1. */
     judgeRepeat?: number;
     /** Unified judge config. Always non-empty; 1 entry = single judge, ≥ 2 = ensemble.

package/dist/cli/lib/parse-run-config.js

@@ -41,10 +41,10 @@ export function parseRunConfig(values) {
         ? loadEvalConfig(values.config)
         : null;
     const skillDir = resolve(values['skill-dir'] ?? 'skills');
-    // 2) Resolve samples path: CLI > config > <skillDir>/<treatment>/.omk/ discovery > cwd default.
-    // The .omk/ discovery only fires when exactly one --treatment is given, so omk knows
-    // which skill's bundled samples to use. The dir form (loadSamples handles both file + dir)
-    // means a skill can split samples across multiple files (workflow.json / platform.json / ...).
+    // 2) Resolve samples path: CLI > config > single-treatment skill-local discovery > cwd project default.
+    // Skill-local discovery only fires when exactly one --treatment is given, so omk knows
+    // which skill's bundled samples to use. The `<skill>/.omk/` dir form (loadSamples handles
+    // both file + dir) means a skill can split samples across multiple files.
     const cliSamples = values.samples;
     let samplesFile;
     if (cliSamples) {

package/dist/cli/lib/resolve-skill-input.js

@@ -1,10 +1,11 @@
 import { resolve, join, dirname, basename } from 'node:path';
 import { existsSync, statSync } from 'node:fs';
 import { tCli } from './i18n.js';
+import { findFlatSkillSamplesPath, findProjectSamplesFile, findSkillSamplesPath, skillLocalSamplesDir, } from '../../inputs/sample-locator.js';
 // 统一 skill 入参解析:既接受 SKILL.md 文件(老式 + flat skill),也接受 directory-skill
-// 目录(skills/foo/ 自动找 foo/SKILL.md)。samples 发现优先级:.omk/ 目录(loadSamples
-// 支持目录模式，自动 glob 多文件) > eval-samples.json > .yaml > .yml,fallback 是
-// .omk/ 目录路径(给上游 "samples 不存在" 的提示一个稳定路径)。
+// 目录(skills/foo/ 自动找 foo/SKILL.md)。目录-skill 以 `<skill>/.omk/` 为标准
+// samples 命名空间;扁平 .md 兼容 paired sidecar,找不到时回到项目级
+// `eval-samples.json`。
 //
 // 错误用 tCli 走 i18n,调用方直接 console.error err.message 给用户看,zh/en 都要正确。
 export function resolveSkillInput(input, lang) {
@@ -26,16 +27,13 @@ export function resolveSkillInput(input, lang) {
         skillPath = resolved;
         skillDir = dirname(resolved);
     }
-    const omkDir = join(skillDir, '.omk');
-    const candidates = [
-        ...(existsSync(omkDir) ? [omkDir] : []),
-        join(skillDir, 'eval-samples.json'),
-        join(skillDir, 'eval-samples.yaml'),
-        join(skillDir, 'eval-samples.yml'),
-    ];
-    // fallback 到 .omk/ 目录：上游会报 "no sample files found in directory" 引导用户创建
-    const samplesPath = candidates.find(existsSync) ?? omkDir;
     // 形态以解析后的 skillPath 命名为准:目录-skill 的 skillPath 总是 `.../SKILL.md`。
     const isDirectorySkill = basename(skillPath) === 'SKILL.md';
+    const flatSkillName = basename(skillPath).replace(/\.md$/i, '');
+    const samplesPath = isDirectorySkill
+        ? findSkillSamplesPath(skillDir) ?? skillLocalSamplesDir(skillDir)
+        : findFlatSkillSamplesPath(skillDir, flatSkillName)
+            ?? findProjectSamplesFile(process.cwd())
+            ?? 'eval-samples.json';
     return { skillPath, skillDir, samplesPath, isDirectorySkill };
 }

package/dist/doctor/messages.js

@@ -133,8 +133,8 @@ export const DOCTOR_MESSAGES = {
         en: 'directory-skill missing SKILL.md entry file',
     },
     'cli.doctor.skill_metadata.hint.frontmatter': {
-        zh: 'front-matter 用 YAML 语法,key: value 或 - item 形式。可参考 examples/multi-skills 下的 skill 写法',
-        en: 'front-matter uses YAML syntax (key: value or - item). See examples/multi-skills for reference',
+        zh: 'front-matter 用 YAML 语法，key: value 或 - item 形式。可参考 examples/skill-map-showcase/skills/release-readiness 的目录式 skill 写法',
+        en: 'front-matter uses YAML syntax (key: value or - item). See examples/skill-map-showcase/skills/release-readiness for a directory-skill reference',
     },
     'cli.doctor.skill_metadata.hint.hardrules': {
         zh: 'hardRules 必须写在 SKILL.md front-matter 中,格式为 hardRules: [{ id, rule, expectedBehavior }]；id 要稳定且唯一,expectedBehavior 写可观察行为。',

package/dist/eval-core/artifact-file-names.d.ts

@@ -0,0 +1,15 @@
+export declare const REPORT_FILE_SUFFIX = ".report.json";
+export declare const GRAPH_FILE_SUFFIX = ".graph.json";
+export declare const CARD_FILE_SUFFIX = ".card.md";
+export declare function safeArtifactFileStem(id: string): string;
+export declare function reportFileName(stem: string): string;
+export declare function reportFilePath(dir: string, stem: string): string;
+export declare function reportFileStem(fileName: string): string | null;
+export declare function isReportFileName(fileName: string): boolean;
+export declare function graphFileName(stem: string): string;
+export declare function cardFileName(stem: string): string;
+export declare function runTimestamp(date?: Date): string;
+export declare function randomRunToken(): string;
+export declare function runFileSuffix(counter?: number): string;
+export declare function stripDomainPrefix(id: string, domain: string): string;
+export declare function doctorReportFileStem(skillName: string, reportId: string): string;

package/dist/eval-core/artifact-file-names.js

@@ -0,0 +1,46 @@
+import { join } from 'node:path';
+export const REPORT_FILE_SUFFIX = '.report.json';
+export const GRAPH_FILE_SUFFIX = '.graph.json';
+export const CARD_FILE_SUFFIX = '.card.md';
+export function safeArtifactFileStem(id) {
+    return id.replaceAll(/[/\\:*?"<>|]/g, '_');
+}
+export function reportFileName(stem) {
+    return `${safeArtifactFileStem(stem)}${REPORT_FILE_SUFFIX}`;
+}
+export function reportFilePath(dir, stem) {
+    return join(dir, reportFileName(stem));
+}
+export function reportFileStem(fileName) {
+    return fileName.endsWith(REPORT_FILE_SUFFIX)
+        ? fileName.slice(0, -REPORT_FILE_SUFFIX.length)
+        : null;
+}
+export function isReportFileName(fileName) {
+    return reportFileStem(fileName) !== null;
+}
+export function graphFileName(stem) {
+    return `${safeArtifactFileStem(stem)}${GRAPH_FILE_SUFFIX}`;
+}
+export function cardFileName(stem) {
+    return `${safeArtifactFileStem(stem)}${CARD_FILE_SUFFIX}`;
+}
+export function runTimestamp(date = new Date()) {
+    const pad = (n) => String(n).padStart(2, '0');
+    return `${date.getFullYear()}${pad(date.getMonth() + 1)}${pad(date.getDate())}T${pad(date.getHours())}${pad(date.getMinutes())}${pad(date.getSeconds())}`;
+}
+export function randomRunToken() {
+    return Math.random().toString(36).slice(2, 6);
+}
+export function runFileSuffix(counter) {
+    const middle = counter === undefined ? '' : `-${counter}`;
+    return `${runTimestamp()}${middle}-${randomRunToken()}`;
+}
+export function stripDomainPrefix(id, domain) {
+    const safeId = safeArtifactFileStem(id);
+    const prefix = `${domain}-`;
+    return safeId.startsWith(prefix) ? safeId.slice(prefix.length) : safeId;
+}
+export function doctorReportFileStem(skillName, reportId) {
+    return `${safeArtifactFileStem(skillName)}-${stripDomainPrefix(reportId, 'doctor')}`;
+}

package/dist/eval-core/evaluation-job.d.ts

@@ -1,5 +1,5 @@
 import type { Artifact, EvaluationErrorCategory, EvaluationJob, EvaluationRequest, EvaluationRun, JudgeConfig } from '../types/index.js';
-export declare function buildEvaluationRequest({ samplesPath, skillDir, artifacts, model, executor, noJudge, concurrency, timeoutMs, noCache, dryRun, project, owner, tags, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }: {
+export declare function buildEvaluationRequest({ samplesPath, skillDir, artifacts, model, executor, noJudge, concurrency, timeoutMs, noCache, dryRun, project, owner, tags, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }: {
     samplesPath: string;
     skillDir: string;
     artifacts: Artifact[];
@@ -14,6 +14,7 @@ export declare function buildEvaluationRequest({ samplesPath, skillDir, artifact
     owner?: string;
     tags?: string[];
     repeat?: number;
+    holdoutRatio?: number;
     batch?: boolean;
     judgeRepeat?: number;
     judgeModels: JudgeConfig[];

package/dist/eval-core/evaluation-job.js

@@ -1,7 +1,7 @@
 function nowIso() {
     return new Date().toISOString();
 }
-export function buildEvaluationRequest({ samplesPath, skillDir, artifacts, model, executor, noJudge, concurrency, timeoutMs, noCache, dryRun, project, owner, tags, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }) {
+export function buildEvaluationRequest({ samplesPath, skillDir, artifacts, model, executor, noJudge, concurrency, timeoutMs, noCache, dryRun, project, owner, tags, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }) {
     return {
         samplesPath,
         skillDir,
@@ -17,6 +17,7 @@ export function buildEvaluationRequest({ samplesPath, skillDir, artifacts, model
         owner,
         tags,
         repeat,
+        holdoutRatio,
         batch,
         judgeRepeat,
         judgeModels,

package/dist/eval-core/evaluation-reporting.js

@@ -5,6 +5,7 @@ import { createHash } from 'node:crypto';
 import { fileURLToPath } from 'node:url';
 import { DEFAULT_REPORTS_DIR } from './default-dirs.js';
 import { indexReportWrite } from './artifact-index.js';
+import { reportFilePath } from './artifact-file-names.js';
 import { buildVariantSummary } from './schema.js';
 import { buildVariantConfig, resolveExecutionStrategy } from './execution-strategy.js';
 import { getJudgePromptHash } from '../grading/judge.js';
@@ -61,10 +62,14 @@ export function getCliVersion() {
     return PKG.version;
 }
 export function getGitInfo() {
+    // stdio 静默 stderr:在非 git 目录(如 omk init 出来的 demo)里 rev-parse 会打印
+    // `fatal: not a git repository` 到终端。catch 已把失败兜成 null(报告省略 git 信息),
+    // 这条 fatal 对用户是纯噪声,吞掉它。与 skill-loader 的 GIT_PROBE_STDIO 同口径。
+    const gitProbeStdio = ['ignore', 'pipe', 'ignore'];
     try {
-        const commit = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf-8' }).trim();
-        const branch = execFileSync('git', ['rev-parse', '--abbrev-ref', 'HEAD'], { encoding: 'utf-8' }).trim();
-        const dirty = execFileSync('git', ['status', '--porcelain'], { encoding: 'utf-8' }).trim().length > 0;
+        const commit = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf-8', stdio: gitProbeStdio }).trim();
+        const branch = execFileSync('git', ['rev-parse', '--abbrev-ref', 'HEAD'], { encoding: 'utf-8', stdio: gitProbeStdio }).trim();
+        const dirty = execFileSync('git', ['status', '--porcelain'], { encoding: 'utf-8', stdio: gitProbeStdio }).trim().length > 0;
         return { commit, commitShort: commit.slice(0, 7), branch, dirty };
     }
     catch {
@@ -276,7 +281,7 @@ export function persistReport(report, outputDir) {
         return null;
     if (!existsSync(outputDir))
         mkdirSync(outputDir, { recursive: true });
-    const filePath = join(outputDir, `${report.id}.json`);
+    const filePath = reportFilePath(outputDir, report.id);
     writeFileSync(filePath, JSON.stringify(report, null, 2));
     // 产物发现索引:报告落项目本地后,best-effort 追加全局轻卡片,让 omk studio 跨项目聚合成机器级总览。
     // 永不抛、永不阻断报告落盘(正文是 source of truth)。

package/dist/eval-core/holdout.d.ts

@@ -0,0 +1,66 @@
+/**
+ * Holdout split + train/holdout composite breakdown.
+ *
+ * Deterministic (no RNG) train/holdout partitioning of a sample set, plus the
+ * subset-composite recompute that lets `omk eval --holdout-ratio` and `omk evolve`
+ * score a variant on a withheld slice using the *same* aggregation as the headline
+ * composite (`buildVariantSummary`). Lives in eval-core so both the eval pipeline
+ * and the authoring/evolve loop depend *down* into it (authoring → eval-core is the
+ * established direction).
+ *
+ * A large train − holdout composite gap is the generalization / sample-set-overfitting
+ * signal the verdict's overfitting gate reads (`src/eval-core/verdict.ts`).
+ */
+import type { Report, HoldoutBreakdown } from '../types/index.js';
+/** A train / holdout partition of a sample set. */
+export interface HoldoutSplit {
+    trainIds: Set<string>;
+    holdoutIds: Set<string>;
+}
+/** Below this many samples on any side, a split is too small to be meaningful —
+ *  callers fall back to full-set scoring and mark the breakdown `disabled`. */
+export declare const MIN_HOLDOUT_SUBSET = 3;
+/** Pick `count` ids at an even stride across `ids` (deterministic, no RNG) so the
+ *  picked subset is representative of the ordering and stable across rounds/runs. */
+export declare function pickByStride(ids: string[], count: number): Set<string>;
+/**
+ * Deterministically split sample ids into train / holdout by `ratio` (fraction
+ * held out). Holdout members are picked at an even stride so the partition is
+ * representative of the ordering, and the split is stable across rounds and runs
+ * (no RNG). Returns null when ratio ≤ 0 or either side would drop below
+ * MIN_HOLDOUT_SUBSET — the caller then scores on the full set.
+ */
+export declare function splitHoldout(sampleIds: string[], ratio: number): HoldoutSplit | null;
+/**
+ * Mean composite over the subset of a report's results whose sample_id is in
+ * `ids`, using the same aggregation as the full-run summary
+ * (`buildVariantSummary`) so train / holdout scores stay comparable to the
+ * headline composite. Returns 0 when the subset has no scorable entries.
+ */
+export declare function subsetCompositeScore(report: Report, variantKey: string, ids: Set<string>): number;
+/**
+ * How many subset results actually produced a usable composite (> 0) for a variant.
+ * `buildVariantSummary` averages only `compositeScore > 0` entries (schema.ts), so
+ * the mean can rest on far fewer samples than the authored split size when runs
+ * flake / partial-error / budget-abort. The overfitting gate must trust THIS count,
+ * not the authored `trainCount` / `holdoutCount`, or a 1-of-3 holdout gets dressed
+ * up as a 3-sample-backed conclusion.
+ */
+export declare function subsetScorableCount(report: Report, variantKey: string, ids: Set<string>): number;
+/**
+ * Train vs holdout composite breakdown per variant for `omk eval --holdout-ratio`.
+ * Post-hoc — never perturbs the headline aggregation or bootstrap CI.
+ *
+ * The split is taken over `sampleIdOrder` — the **stable authored sample order**
+ * (the loaded `samples` file order), NOT `report.results`, whose insertion order
+ * is the concurrent-completion order and drifts run-to-run. Binding the stride pick
+ * to the authored order is what makes the holdout (and the verdict overfitting gate
+ * it feeds) deterministic and reproducible. Subset scores are then read from
+ * `report.results` by id-set membership, which is order-independent.
+ *
+ * When the split is too small on either side (< MIN_HOLDOUT_SUBSET) it returns
+ * `{ disabled: true }` with an empty `perVariant`, so the verdict overfitting gate
+ * stays inert. The testSetHash watermark (gap-spec §7.1) is attached by the caller,
+ * shared with gapReports.
+ */
+export declare function computeHoldoutBreakdown(report: Report, variantNames: string[], ratio: number, sampleIdOrder: string[]): HoldoutBreakdown;