oh-my-knowledge 0.41.0 → 0.43.0

package/dist/authoring/evolver.js

@@ -8,7 +8,7 @@ import { projectReportsDir, globalReportsDir } from '../eval-core/measurement-di
 import { analyzeResults } from '../analysis/report-diagnostics.js';
 import { loadSamples } from '../inputs/load-samples.js';
 import { hashArtifactSource } from '../inputs/content-hash.js';
-import { buildVariantSummary } from '../eval-core/schema.js';
+import { MIN_HOLDOUT_SUBSET, pickByStride, splitHoldout, subsetCompositeScore } from '../eval-core/holdout.js';
 import { bootstrapDiffCI, DEFAULT_BOOTSTRAP_ALPHA, DEFAULT_BOOTSTRAP_SAMPLES } from '../eval-core/bootstrap.js';
 import { fixSamples } from './sample-fixer.js';
 const IMPROVE_SYSTEM_PROMPT = `你是一个 AI 提示词改进专家。你的任务是分析评测结果中的薄弱环节，针对性地改进 skill（系统提示词），使其在评测中获得更高的分数。
@@ -165,43 +165,11 @@ export function extractWeakSamples(report, variantKey, count = 5, sampleIdFilter
         .sort((a, b) => a.compositeScore - b.compositeScore)
         .slice(0, count);
 }
-/** Below this many samples on any side, a split is too small to be meaningful —
- *  evolve falls back to full-set scoring and warns. */
-const MIN_HOLDOUT_SUBSET = 3;
 /** Below this many decision (val) samples the bootstrap diff CI almost never
  *  excludes 0 for realistic effect sizes, so the significance gate would reject
  *  every candidate. Under that floor evolve degrades to the point-estimate accept
  *  and flags `gate.underpowered`. */
 export const MIN_GATE_SAMPLES = 8;
-/** Pick `count` ids at an even stride across `ids` (deterministic, no RNG) so the
- *  picked subset is representative of the ordering and stable across rounds/runs. */
-function pickByStride(ids, count) {
-    const picked = new Set();
-    if (count <= 0)
-        return picked;
-    const stride = ids.length / count;
-    for (let k = 0; k < count; k++)
-        picked.add(ids[Math.floor(k * stride)]);
-    return picked;
-}
-/**
- * Deterministically split sample ids into train / holdout by `ratio` (fraction
- * held out). Holdout members are picked at an even stride so the partition is
- * representative of the ordering, and the split is stable across rounds and runs
- * (no RNG). Returns null when ratio ≤ 0 or either side would drop below
- * MIN_HOLDOUT_SUBSET — the caller then scores on the full set.
- */
-export function splitHoldout(sampleIds, ratio) {
-    if (!(ratio > 0) || sampleIds.length === 0)
-        return null;
-    const holdoutCount = Math.round(sampleIds.length * ratio);
-    const trainCount = sampleIds.length - holdoutCount;
-    if (holdoutCount < MIN_HOLDOUT_SUBSET || trainCount < MIN_HOLDOUT_SUBSET)
-        return null;
-    const holdoutIds = pickByStride(sampleIds, holdoutCount);
-    const trainIds = new Set(sampleIds.filter((id) => !holdoutIds.has(id)));
-    return { trainIds, holdoutIds };
-}
 /**
  * Deterministically split sample ids into train / val / test. `val` is carved
  * first at an even stride; `test` is carved at an even stride over what remains,
@@ -223,25 +191,6 @@ export function splitTrainValTest(sampleIds, valRatio, testRatio) {
     const trainIds = new Set(sampleIds.filter((id) => !valIds.has(id) && !testIds.has(id)));
     return { trainIds, valIds, testIds };
 }
-/**
- * Mean composite over the subset of a report's results whose sample_id is in
- * `ids`, using the same aggregation as the full-run summary
- * (`buildVariantSummary`) so train / holdout scores stay comparable to the
- * headline composite. Returns 0 when the subset has no scorable entries.
- */
-function subsetCompositeScore(report, variantKey, ids) {
-    const entries = [];
-    for (const r of report.results) {
-        if (!ids.has(r.sample_id))
-            continue;
-        const v = r.variants[variantKey];
-        if (v)
-            entries.push(v);
-    }
-    if (entries.length === 0)
-        return 0;
-    return buildVariantSummary(entries).avgCompositeScore ?? 0;
-}
 /**
  * Per-sample composite scores over the subset of a report's results whose
  * sample_id is in `ids`, in result order. Feeds `bootstrapDiffCI` for the

package/dist/authoring/generator.d.ts

@@ -31,6 +31,30 @@ export declare function generateSamples({ skillContent, count, model, executorNa
     costUSD: number;
 }>;
 type TraceSignalItem = Pick<ObservationInboxItem, 'skillName' | 'signalType' | 'signalSubtype' | 'severity' | 'evidence' | 'messageWindow' | 'occurrences'>;
+export interface StratifiedTraceSignal extends TraceSignalItem {
+    /** Share of total occurrences across all signals (0-1) — drives proportional
+     *  sample allocation in the prompt. */
+    weight: number;
+}
+/**
+ * Rank trace signals by frequency and annotate each with its share of the total
+ * occurrences. Lets `omk sample --from-traces` allocate samples *proportional to
+ * how often a failure actually happened* instead of a flat "1-2 per signal" — a
+ * failure seen 100× deserves more regression coverage than one seen twice.
+ *
+ * Deliberately does NOT re-merge signals here. The observation inbox is the only
+ * source of these items and already aggregates `occurrences` by the FULL identity
+ * — `skillName + cwd + sourceKind + signalType + signalSubtype + evidence`
+ * (`inbox.ts` `keyFor`). Re-merging on a narrower key (e.g. type+subtype+evidence)
+ * would fold *different skills / cwd* with the same failure shape into one entry,
+ * mis-attributing the summed occurrences to the first skill and skewing the
+ * regenerated distribution. So we trust the upstream dedup and only sort + weight.
+ *
+ * Does NOT fix the underlying selection bias (traces only capture *failures*),
+ * which is why the `omk sample --from-traces` draft warning still stands — this
+ * only makes the within-failure distribution representative of frequency.
+ */
+export declare function stratifyTraceSignals(items: TraceSignalItem[]): StratifiedTraceSignal[];
 /**
  * Build the generation prompt for `omk sample --from-traces`. Renders each
  * observation-inbox signal (evidence + message window) into a section and asks

package/dist/authoring/generator.js

@@ -465,10 +465,34 @@ async function finalizeSamples(samples, costUSD, skillContent) {
 const TRACE_GEN_INSTRUCTIONS = `下面给出的不是 skill，而是从生产会话 trace 中观测到的失败 / 异常信号。请为这些信号生成评测用例（eval samples），使评测能复现并守住这些失败模式——把线上真实发生过的问题沉淀成回归用例。
 
 要求：
-- 每个信号生成 1-2 条聚焦回归用例；信号若是噪声 / 证据不足 / 无法复现，跳过它，不要硬凑。
+- 按各信号标注的「占比」分配用例数：高频信号多生成、低频少生成，让用例集覆盖线上失败的真实频次分布（高占比信号 2-3 条，低占比 1 条即可）；信号若是噪声 / 证据不足 / 无法复现，跳过它，不要硬凑。
 - prompt 要还原触发该信号的场景（自然语言任务），不要直接复述证据文本。
 - 断言优先用 mock_hit / tools_called / tools_not_called / tool_input_contains 精确锚定失败步骤，再用 contains 兜底；按「原子型」配比处理（无需工作流编号步骤），除非证据明显是多步流程。
 - 不要在输出里说明判断过程，直接输出 JSON 数组。`;
+/**
+ * Rank trace signals by frequency and annotate each with its share of the total
+ * occurrences. Lets `omk sample --from-traces` allocate samples *proportional to
+ * how often a failure actually happened* instead of a flat "1-2 per signal" — a
+ * failure seen 100× deserves more regression coverage than one seen twice.
+ *
+ * Deliberately does NOT re-merge signals here. The observation inbox is the only
+ * source of these items and already aggregates `occurrences` by the FULL identity
+ * — `skillName + cwd + sourceKind + signalType + signalSubtype + evidence`
+ * (`inbox.ts` `keyFor`). Re-merging on a narrower key (e.g. type+subtype+evidence)
+ * would fold *different skills / cwd* with the same failure shape into one entry,
+ * mis-attributing the summed occurrences to the first skill and skewing the
+ * regenerated distribution. So we trust the upstream dedup and only sort + weight.
+ *
+ * Does NOT fix the underlying selection bias (traces only capture *failures*),
+ * which is why the `omk sample --from-traces` draft warning still stands — this
+ * only makes the within-failure distribution representative of frequency.
+ */
+export function stratifyTraceSignals(items) {
+    const total = items.reduce((sum, it) => sum + (it.occurrences ?? 0), 0) || 1;
+    return items
+        .map((it) => ({ ...it, occurrences: it.occurrences ?? 0, weight: (it.occurrences ?? 0) / total }))
+        .sort((a, b) => b.occurrences - a.occurrences);
+}
 /**
  * Build the generation prompt for `omk sample --from-traces`. Renders each
  * observation-inbox signal (evidence + message window) into a section and asks
@@ -477,7 +501,9 @@ const TRACE_GEN_INSTRUCTIONS = `下面给出的不是 skill，而是从生产会
  * judge prompt — so judge-prompt isolation is unaffected.
  */
 export function buildSamplesFromTracesPrompt(items, count) {
-    const sections = items.map((it, i) => {
+    // 先按频次分层（合并重复 + 算占比 + 降序），让模型按「占比」分配配额，而非每信号一刀切。
+    const stratified = stratifyTraceSignals(items);
+    const sections = stratified.map((it, i) => {
         const ev = it.evidence ?? {};
         const evLines = [
             ev.tool && `工具: ${ev.tool}`,
@@ -491,14 +517,15 @@ export function buildSamplesFromTracesPrompt(items, count) {
             ? '\n上下文消息:\n' + [...it.messageWindow.before, ...it.messageWindow.event, ...it.messageWindow.after]
                 .map((m) => `  [${m.role}] ${m.snippet}`).join('\n')
             : '';
-        return `### 信号 ${i + 1}：${it.signalType} / ${it.signalSubtype}（严重度 ${it.severity}，出现 ${it.occurrences} 次，skill: ${it.skillName}）\n${evLines}${win}`;
+        const pct = (it.weight * 100).toFixed(0);
+        return `### 信号 ${i + 1}：${it.signalType} / ${it.signalSubtype}（严重度 ${it.severity}，出现 ${it.occurrences} 次 · 占比 ${pct}%，skill: ${it.skillName}）\n${evLines}${win}`;
     }).join('\n\n---\n\n');
     const countLine = typeof count === 'number'
-        ? `共生成约 ${count} 条评测用例。`
-        : '按上面的「每个信号 1-2 条」生成。';
+        ? `共生成约 ${count} 条评测用例，按各信号的「占比」分配配额（高频多、低频少），覆盖整体失败分布。`
+        : '按各信号「占比」分配：高频信号多生成、低频少生成，覆盖整体失败分布。';
     return `${TRACE_GEN_INSTRUCTIONS}
 
-## 观测到的失败信号（共 ${items.length} 个）
+## 观测到的失败信号（共 ${stratified.length} 个，已按出现频次降序）
 
 ${sections}
 

package/dist/cli/commands/doctor.js

@@ -1,5 +1,5 @@
-import { existsSync, statSync, mkdirSync, readFileSync, readdirSync, unlinkSync, writeFileSync } from 'node:fs';
-import { dirname, join, resolve } from 'node:path';
+import { mkdirSync, readFileSync, readdirSync, unlinkSync, writeFileSync } from 'node:fs';
+import { join, resolve } from 'node:path';
 import { Args, Flags } from '@oclif/core';
 import { LANG_FLAG, bilingual } from '../oclif/i18n.js';
 import { BaseCommand } from '../oclif/base-command.js';
@@ -9,50 +9,11 @@ import { tCli } from '../lib/i18n.js';
 import { makeDoctorProgress } from '../lib/progress.js';
 import { DEFAULT_DOCTORS_DIR } from '../../eval-core/default-dirs.js';
 import { indexDoctorWrite, removeDoctorCard } from '../../eval-core/artifact-index.js';
+import { doctorReportFileStem, isReportFileName, reportFilePath } from '../../eval-core/artifact-file-names.js';
+import { migrateLegacyReportFiles } from '../../eval-core/report-file-migration.js';
 import { projectDoctorsDir, globalDoctorsDir } from '../../eval-core/measurement-dirs.js';
-const DEFAULT_SAMPLE_FILENAMES = ['eval-samples.json', 'eval-samples.yaml', 'eval-samples.yml'];
-function findSamplesInDir(dir) {
-    for (const name of DEFAULT_SAMPLE_FILENAMES) {
-        const candidate = join(dir, name);
-        if (existsSync(candidate))
-            return candidate;
-    }
-    return null;
-}
-function sampleSearchDirs(target, cwd) {
-    const dirs = [];
-    const add = (dir) => {
-        const abs = resolve(dir);
-        if (!dirs.includes(abs))
-            dirs.push(abs);
-    };
-    if (target) {
-        const absTarget = resolve(target);
-        if (existsSync(absTarget)) {
-            const stat = statSync(absTarget);
-            if (stat.isDirectory()) {
-                add(absTarget);
-                add(dirname(absTarget));
-                add(dirname(dirname(absTarget)));
-            }
-            else {
-                const parent = dirname(absTarget);
-                add(parent);
-                add(dirname(parent));
-            }
-        }
-    }
-    add(cwd);
-    return dirs;
-}
-function findDefaultSamplesPath(target, cwd) {
-    for (const dir of sampleSearchDirs(target, cwd)) {
-        const samplesPath = findSamplesInDir(dir);
-        if (samplesPath)
-            return samplesPath;
-    }
-    return null;
-}
+import { findDoctorDeprecatedSamplesHint, findDoctorSamplesPath } from '../../inputs/sample-locator.js';
+import { persistDoctorGraphSidecars, removeDoctorGraphSidecars } from '../../artifact-graph/doctor.js';
 export default class Doctor extends BaseCommand {
     static description = bilingual({
         zh: '体检 omk 工作目录，检查 skill 配置 / 依赖 / executor 连通性。',
@@ -186,7 +147,16 @@ export default class Doctor extends BaseCommand {
             const timeoutSec = flags.timeout != null ? Number(flags.timeout) : defaultTimeoutSec;
             const timeoutMs = Math.max(1000, Math.floor((Number.isFinite(timeoutSec) ? timeoutSec : defaultTimeoutSec) * 1000));
             const cwd = process.cwd();
-            const samplesPath = flags.samples ? resolve(flags.samples) : findDefaultSamplesPath(target, cwd);
+            const samplesPath = flags.samples ? resolve(flags.samples) : findDoctorSamplesPath(target, cwd);
+            const deprecatedSamplesHint = !flags.samples && !samplesPath
+                ? findDoctorDeprecatedSamplesHint(target, cwd)
+                : null;
+            if (deprecatedSamplesHint) {
+                process.stderr.write(tCli('cli.common.deprecated_skill_samples_path', lang, {
+                    oldPath: deprecatedSamplesHint.oldPath,
+                    newPath: deprecatedSamplesHint.newPath,
+                }));
+            }
             let samples;
             let requires;
             if (samplesPath) {
@@ -270,7 +240,7 @@ export default class Doctor extends BaseCommand {
             }
             persistDoctorReport(report, flags['output-dir']
                 ? resolve(flags['output-dir'])
-                : (flags.global ? globalDoctorsDir() : projectDoctorsDir()));
+                : (flags.global ? globalDoctorsDir() : projectDoctorsDir()), lang);
             if (flags.fix) {
                 const existing = report;
                 if (existing.outcome !== 'failed') {
@@ -288,47 +258,76 @@ export default class Doctor extends BaseCommand {
 // 每个 skill 最多保留多少份历史 doctor 报告(避免无界增长拖慢 studio 启动 +
 // scanDoctorReports 扫盘成本)。50 = ~每天 1 跑撑 1.5 个月 sparkline,够用。
 const DOCTOR_HISTORY_MAX_PER_SKILL = 50;
-function persistDoctorReport(report, outputDir) {
+function persistDoctorReport(report, outputDir, lang = 'zh') {
     const dir = outputDir ?? DEFAULT_DOCTORS_DIR;
     mkdirSync(dir, { recursive: true });
-    const safeId = report.id.replace(/[/\\:*?"<>|]/g, '_');
+    migrateLegacyReportFiles(dir, 'doctor');
     for (const skill of report.skills) {
-        const counts = { pass: 0, warn: 0, fail: 0, skipped: 0 };
+        const counts = {
+            pass: 0,
+            warn: 0,
+            fail: 0,
+            skipped: 0,
+        };
         for (const r of skill.results) {
             const s = r.status;
             if (s in counts)
                 counts[s]++;
         }
+        const outcome = skill.status === 'fail' ? 'failed' : skill.status === 'warn' ? 'warnings_only' : 'passed';
         const perSkill = {
             ...report,
             skills: [skill],
-            ruleStats: { ...counts, total: skill.results.length },
+            ruleStats: {
+                pass: counts.pass,
+                warn: counts.warn,
+                fail: counts.fail,
+                skipped: counts.skipped,
+                total: skill.results.length,
+            },
             totals: {
                 pass: skill.status === 'pass' ? 1 : 0,
                 warn: skill.status === 'warn' ? 1 : 0,
                 fail: skill.status === 'fail' ? 1 : 0,
             },
-            outcome: skill.status === 'fail' ? 'failed' : skill.status === 'warn' ? 'warnings_only' : 'passed',
+            outcome,
         };
-        const safeName = skill.skillName.replace(/[/\\:*?"<>|]/g, '_');
-        const cardId = `${safeName}-${safeId}`;
-        const filePath = join(dir, `${cardId}.json`);
+        const cardId = doctorReportFileStem(skill.skillName, report.id);
+        const filePath = reportFilePath(dir, cardId);
         writeFileSync(filePath, JSON.stringify(perSkill, null, 2), 'utf8');
         // 产物发现索引:per-skill 报告落项目本地后,best-effort 追加全局轻卡片,让 studio 跨项目聚合。
         indexDoctorWrite({
             id: cardId, path: filePath, skillName: skill.skillName, reportId: report.id, timestamp: report.timestamp,
             status: skill.status, passCount: counts.pass, warnCount: counts.warn, failCount: counts.fail,
         }, dir);
+        try {
+            persistDoctorGraphSidecars({
+                report: perSkill,
+                skill,
+                sourcePath: filePath,
+                outputDir: dir,
+                fileStem: cardId,
+                lang,
+            });
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            const warning = lang === 'zh'
+                ? `⚠️  doctor graph sidecar 写入失败：${message}\n`
+                : `⚠️  failed to write doctor graph sidecar: ${message}\n`;
+            process.stderr.write(warning);
+        }
         pruneDoctorHistory(dir, skill.skillName, DOCTOR_HISTORY_MAX_PER_SKILL);
     }
 }
-// 写入新报告后调用:扫 dir 里属于该 skill 的所有 single-skill doctor JSON,
+// 写入新报告后调用:扫 dir 里属于该 skill 的所有 single-skill doctor report,
 // 按 timestamp 倒排,保留 maxKeep 份最近的,其余删。按 content 匹配 skillName 不
-// 看文件名,所以同时清理新 `{name}-{id}.json` 与遗留 `{name}.json` 两种命名。
+// 看文件名,所以清理逻辑不依赖 readdir 顺序或 stem 推断 skill 名。
 export function pruneDoctorHistory(dir, skillName, maxKeep) {
+    migrateLegacyReportFiles(dir, 'doctor');
     const candidates = [];
     for (const file of readdirSync(dir)) {
-        if (!file.endsWith('.json'))
+        if (!isReportFileName(file))
             continue;
         try {
             const data = JSON.parse(readFileSync(join(dir, file), 'utf-8'));
@@ -337,20 +336,22 @@ export function pruneDoctorHistory(dir, skillName, maxKeep) {
                 continue;
             if (data.skills[0].skillName !== skillName)
                 continue;
-            candidates.push({ file, timestamp: data.timestamp });
+            candidates.push({ file, graphStem: doctorReportFileStem(skillName, data.id), timestamp: data.timestamp });
         }
         catch { /* skip corrupt / unrelated json */ }
     }
     if (candidates.length <= maxKeep)
         return;
     candidates.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
-    for (const { file } of candidates.slice(maxKeep)) {
+    for (const { file, graphStem } of candidates.slice(maxKeep)) {
         try {
             unlinkSync(join(dir, file));
         }
         catch { /* ignore */ }
         // 连带删卡片:否则被 prune 掉的报告会经 listDoctorCards 合并在本项目 studio「复活」(正文已删、卡片还在)。
         // 卡片 id = 文件 stem(`{name}-{id}`),与 indexDoctorWrite 写入口径一致。
-        removeDoctorCard(file.replace(/\.json$/, ''));
+        const doctorStem = file.replace(/\.report\.json$/, '');
+        removeDoctorCard(doctorStem);
+        removeDoctorGraphSidecars(dir, graphStem);
     }
 }

package/dist/cli/commands/eval/index.d.ts

@@ -38,6 +38,7 @@ export default class Eval extends BaseCommand {
         effort: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
         'no-diagnostic': import("@oclif/core/interfaces").BooleanFlag<boolean>;
         repeat: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
+        'holdout-ratio': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
         'judge-repeat': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
         bootstrap: import("@oclif/core/interfaces").BooleanFlag<boolean>;
         'bootstrap-samples': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;

package/dist/cli/commands/eval/index.js

@@ -10,6 +10,7 @@ import { computeRunTally } from '../../lib/run-tally.js';
 import { DEFAULT_BOOTSTRAP_SAMPLES } from '../../../eval-core/bootstrap.js';
 import { DEFAULT_GATE_THRESHOLD } from '../../../eval-core/verdict.js';
 import { EVALUATION_REPORT_SCHEMA_VERSION } from '../../../eval-core/evaluation-reporting.js';
+import { findSingleTreatmentDeprecatedSamplesHint, hasUsableSamplesPath, } from '../../../inputs/sample-locator.js';
 function isDryRunReport(report) {
     return Boolean(report && typeof report === 'object' && report.dryRun === true);
 }
@@ -41,10 +42,30 @@ function applyGateExitCode(code, values, lang) {
     process.stderr.write(tCli('cli.run.report_only_gate_skipped', lang));
     return 0;
 }
+/**
+ * 完整 report JSON 是**机器输出**:重定向 / 管道(`omk eval > r.json`、`| jq`)时吐到 stdout 供下游消费。
+ * 交互式 TTY 下报告已存盘、(默认)还起了 report server,再刷上千行 JSON 只会把 verdict 淹没在屏幕外 ——
+ * 故只在非 TTY(stdout 被重定向 / 管道)时 dump。dry-run 的 JSON 是用户显式索取的产物,不走此门控。
+ */
+function emitReportJson(report) {
+    if (!process.stdout.isTTY) {
+        console.log(JSON.stringify(report, null, 2));
+    }
+}
+/**
+ * 给人读的 verdict 文案:stdout 是 TTY 时进 stdout(交互终端没有 JSON,verdict 就是答案),
+ * 否则进 stderr —— 与 emitReportJson 配对,保证非 TTY 的 stdout 是**纯 report JSON**,
+ * `omk eval | jq` / `> report.json` 不会被末尾拼上的人类文案噎住(否则 JSON.parse 直接失败)。
+ */
+function emitVerdictText(text) {
+    // 与 console.log 等价(对单个字符串 = write(text + '\n')),只切换目标流,逐字节保留既有文案。
+    const stream = process.stdout.isTTY ? process.stdout : process.stderr;
+    stream.write(text + '\n');
+}
 async function emitEvaluationVerdict(report, values, lang) {
     const { computeVerdict, formatVerdictText } = await import('../../../eval-core/verdict.js');
     const result = computeVerdict(report, verdictOptions(values));
-    console.log(formatVerdictText(result, { verbose: true }));
+    emitVerdictText(formatVerdictText(result, { verbose: true, lang }));
     await recordEvidenceSafely(report, result.level, values, lang);
     return verdictPasses(result.level, result.headline) ? 0 : 1;
 }
@@ -123,13 +144,13 @@ async function emitBatchVerdict(report, reportsDir, values, lang) {
     const status = lang === 'zh'
         ? (failed === 0 ? '通过' : '未通过')
         : (failed === 0 ? 'PASS' : 'FAIL');
-    console.log(tCli('cli.run.batch_verdict_header', lang, {
+    emitVerdictText(tCli('cli.run.batch_verdict_header', lang, {
         status,
         passed,
         total: results.length,
     }));
     for (const result of results) {
-        console.log(`  ${result.verdict.level}: ${result.treatment} — ${result.verdict.headline}`);
+        emitVerdictText(`  ${result.verdict.level}: ${result.treatment} — ${result.verdict.headline}`);
     }
     return failed === 0 ? 0 : 1;
 }
@@ -160,6 +181,23 @@ async function announceSavedReport({ report, filePath, reportsDir, values, lang,
 }
 async function runEval(_args, flags, lang) {
     const { values, config, evalConfig } = parseRunConfig({ ...flags });
+    if (!values.batch && !hasUsableSamplesPath(config.samplesPath)) {
+        const treatmentRaw = typeof values.treatment === 'string' ? values.treatment : '';
+        const treatments = treatmentRaw.split(',').map((v) => v.trim()).filter(Boolean);
+        const deprecatedSamplesHint = !values.samples && !evalConfig?.samples && treatments.length === 1
+            ? findSingleTreatmentDeprecatedSamplesHint(treatments[0], config.skillDir, process.cwd())
+            : null;
+        if (deprecatedSamplesHint) {
+            process.stderr.write(tCli('cli.common.deprecated_skill_samples_path', lang, {
+                oldPath: deprecatedSamplesHint.oldPath,
+                newPath: deprecatedSamplesHint.newPath,
+            }));
+        }
+        console.error(tCli('cli.common.error_prefix', lang, {
+            message: tCli('cli.common.samples_not_found', lang, { path: config.samplesPath }),
+        }));
+        throw new CliExit(1);
+    }
     const { runEvaluation, runMultiple, runBatchEvaluation } = await import('../../../eval-workflows/run-evaluation.js');
     config.onProgress = makeOnProgress(lang);
     const repeatRaw = values.repeat;
@@ -168,6 +206,13 @@ async function runEval(_args, flags, lang) {
         process.stderr.write(tCli('cli.run.invalid_repeat', lang, { value: repeatRaw }));
     }
     const repeatCount = Math.max(1, Math.floor(parsedRepeat) || 1);
+    const holdoutRatioRaw = values['holdout-ratio'];
+    const parsedHoldoutRatio = holdoutRatioRaw !== undefined ? Number(holdoutRatioRaw) : (evalConfig?.holdoutRatio ?? 0);
+    if (holdoutRatioRaw !== undefined && (!Number.isFinite(parsedHoldoutRatio) || parsedHoldoutRatio <= 0 || parsedHoldoutRatio >= 1)) {
+        process.stderr.write(tCli('cli.run.invalid_holdout_ratio', lang, { value: holdoutRatioRaw }));
+    }
+    if (parsedHoldoutRatio > 0 && parsedHoldoutRatio < 1)
+        config.holdoutRatio = parsedHoldoutRatio;
     const judgeRepeatRaw = values['judge-repeat'];
     const parsedJudgeRepeat = judgeRepeatRaw !== undefined ? Number(judgeRepeatRaw) : (evalConfig?.judgeRepeat ?? 1);
     if (judgeRepeatRaw !== undefined && (!Number.isFinite(parsedJudgeRepeat) || parsedJudgeRepeat < 1)) {
@@ -224,11 +269,12 @@ async function runEval(_args, flags, lang) {
                     }
                 },
             });
-            console.log(JSON.stringify(report, null, 2));
             if (isDryRunBatchReport(report)) {
+                console.log(JSON.stringify(report, null, 2));
                 console.log(tCli('cli.run.dry_run_no_scores', lang));
                 throw new CliExit(0);
             }
+            emitReportJson(report);
             if (filePath) {
                 await announceSavedReport({ report, filePath, reportsDir: config.outputDir, values, lang });
             }
@@ -285,7 +331,7 @@ async function runEval(_args, flags, lang) {
                 }
             }
         }
-        console.log(JSON.stringify(report, null, 2));
+        emitReportJson(report);
         if (filePath) {
             await announceSavedReport({ report, filePath, reportsDir: config.outputDir, values, lang });
         }
@@ -349,8 +395,8 @@ export default class Eval extends BaseCommand {
         }),
         samples: Flags.string({
             description: bilingual({
-                zh: '用例文件路径。默认 eval-samples.json，也接受 .yaml/.yml；自动发现 --skill-dir 下的 <skill>/.omk/samples.json。',
-                en: 'Samples file path. Defaults to eval-samples.json (also .yaml/.yml); auto-discovers <skill>/.omk/samples.json under --skill-dir.',
+                zh: '用例文件路径。默认项目级 eval-samples.json，也接受 .yaml/.yml；单 treatment 时可自动发现 <skill>/.omk/。',
+                en: 'Samples path. Defaults to project-level eval-samples.json (also .yaml/.yml); single-treatment runs can auto-discover <skill>/.omk/.',
             }),
         }),
         'skill-dir': Flags.string({
@@ -457,6 +503,13 @@ export default class Eval extends BaseCommand {
             description: bilingual({ zh: '每个 sample 重复跑 N 次', en: 'Repeat each sample N times' }),
             parse: integerStringParser('--repeat', { min: 1 }),
         }),
+        'holdout-ratio': Flags.string({
+            description: bilingual({
+                zh: '留出比例 0-1（如 0.3）；切出 holdout 子集，对比 train/holdout 综合分检测过拟合',
+                en: 'Holdout fraction 0-1 (e.g. 0.3); splits a holdout subset, compares train/holdout composite to flag overfitting',
+            }),
+            parse: numberStringParser('--holdout-ratio', { min: 0, max: 1 }),
+        }),
         'judge-repeat': Flags.string({
             description: bilingual({ zh: '每个 dim 评 N 次', en: 'Judge each dim N times' }),
             parse: integerStringParser('--judge-repeat', { min: 1 }),

package/dist/cli/commands/init.js

@@ -8,21 +8,26 @@ import { tCli } from '../lib/i18n.js';
 const INIT_OMK_GITIGNORE = `# omk 测量 bulk + doctor --fix 备份(项目本地)——不入库;前导 / 锚定 .omk/ 顶层,不误伤嵌套同名目录。
 /observe-health/
 /doctors/
+/graphs/
 /observe-inbox/
 /reports/
 /backups/
 `;
+// 脚手架用例必须过 omk 自身的断言合规校验(load-samples.ts Rule A),否则新用户照
+// 快速开始跑的第一条 omk eval 会直接硬报错。约束:contains / not_contains 的 value
+// 只能是单个 ASCII token(长度 [2,40]、无内部空白、无 CJK);多词 / 中文语义匹配一律
+// 走 rubric 交评委判;regex pattern 不能含 CJK。改这里前先跑 `omk eval --dry-run`
+// (非 lenient 合规 oracle)与 test/cli/init-scaffold-conformance 回归测试。
 const INIT_SAMPLES = `[
   {
     "sample_id": "s001",
     "prompt": "审查以下代码",
     "context": "function authenticate(username, password) {\\n  const query = \`SELECT * FROM users WHERE name='\${username}' AND pass='\${password}'\`;\\n  return db.execute(query);\\n}",
-    "rubric": "应识别 SQL 注入风险，建议使用参数化查询",
+    "rubric": "应识别 SQL 注入风险，建议使用参数化查询；不应把这段代码判为安全无问题。",
     "assertions": [
       { "type": "contains", "value": "SQL", "weight": 1 },
       { "type": "contains", "value": "injection", "weight": 1 },
-      { "type": "regex", "pattern": "parameterized|prepared|placeholder|bind", "flags": "i", "weight": 0.5 },
-      { "type": "not_contains", "value": "looks good", "weight": 0.5 }
+      { "type": "regex", "pattern": "parameterized|prepared|placeholder|bind", "flags": "i", "weight": 0.5 }
     ],
     "dimensions": {
       "security": "是否准确识别出 SQL 注入漏洞并说明其危害",
@@ -35,8 +40,7 @@ const INIT_SAMPLES = `[
     "context": "async function fetchData(url) {\\n  const res = await fetch(url);\\n  const data = await res.json();\\n  return data;\\n}",
     "rubric": "应指出缺少错误处理（网络异常、非 JSON 响应、HTTP 错误状态码）",
     "assertions": [
-      { "type": "contains", "value": "error handling", "weight": 1 },
-      { "type": "regex", "pattern": "try[\\\\s\\\\S]*catch|exception|error", "flags": "i", "weight": 1 },
+      { "type": "regex", "pattern": "try[\\\\s\\\\S]*catch|catch|exception|error", "flags": "i", "weight": 1 },
       { "type": "contains", "value": "status", "weight": 0.5 }
     ],
     "dimensions": {
@@ -154,9 +158,9 @@ export default class Init extends BaseCommand {
             console.log(tCli('cli.init.scaffolded', lang, { dir: targetDir }));
             console.log('');
             console.log(tCli('cli.init.next_steps_title', lang));
-            console.log(tCli('cli.init.next_step_edit_samples', lang));
-            console.log(tCli('cli.init.next_step_edit_skills', lang));
             console.log(tCli('cli.init.next_step_run', lang));
+            console.log(tCli('cli.init.next_step_executor', lang));
+            console.log(tCli('cli.init.next_step_customize', lang));
             console.log(tCli('cli.init.note_codex_executor', lang));
         });
     }

package/dist/cli/commands/observe/index.d.ts

@@ -2,8 +2,8 @@ import { BaseCommand } from '../../oclif/base-command.js';
 import type { SkillHealthReport } from '../../../observability/skill-health-analyzer.js';
 /**
  * observe-health 报告落盘:id / 文件名加 4 位随机段,根治「同秒两次 omk observe 直接覆盖、数据丢失」的 bug。
- * 保留 `-observe-health.json` 后缀 —— resolveObserveHealthDir 靠它判项目优先、listAnalyses 靠它取 id stem、
- * loadAnalysis 靠 `${id}.json` 读真身。落盘后 best-effort 追加全局轻卡片,让 studio 跨项目聚合。
+ * 文件名统一为 `{run}.report.json`:目录表达 observe-health 域,文件表达这是可读报告。
+ * 落盘后 best-effort 追加全局轻卡片,让 studio 跨项目聚合。
  */
 export declare function persistObserveHealthReport(report: SkillHealthReport, outDir: string): {
     id: string;

package/dist/cli/commands/observe/index.js

@@ -1,5 +1,5 @@
 import { mkdirSync, writeFileSync } from 'node:fs';
-import { resolve, join } from 'node:path';
+import { resolve } from 'node:path';
 import { Args, Flags } from '@oclif/core';
 import { LANG_FLAG, bilingual } from '../../oclif/i18n.js';
 import { BaseCommand } from '../../oclif/base-command.js';
@@ -8,17 +8,18 @@ import { tCli } from '../../lib/i18n.js';
 import { parseLastWindow } from '../../lib/shared.js';
 import { projectObserveHealthDir, globalObserveHealthDir } from '../../../eval-core/measurement-dirs.js';
 import { indexObserveWrite } from '../../../eval-core/artifact-index.js';
+import { reportFilePath, runFileSuffix } from '../../../eval-core/artifact-file-names.js';
+import { migrateLegacyReportFiles } from '../../../eval-core/report-file-migration.js';
 /**
  * observe-health 报告落盘:id / 文件名加 4 位随机段,根治「同秒两次 omk observe 直接覆盖、数据丢失」的 bug。
- * 保留 `-observe-health.json` 后缀 —— resolveObserveHealthDir 靠它判项目优先、listAnalyses 靠它取 id stem、
- * loadAnalysis 靠 `${id}.json` 读真身。落盘后 best-effort 追加全局轻卡片,让 studio 跨项目聚合。
+ * 文件名统一为 `{run}.report.json`:目录表达 observe-health 域,文件表达这是可读报告。
+ * 落盘后 best-effort 追加全局轻卡片,让 studio 跨项目聚合。
  */
 export function persistObserveHealthReport(report, outDir) {
     mkdirSync(outDir, { recursive: true });
-    const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
-    const rand = Math.random().toString(36).slice(2, 6);
-    const id = `${timestamp}-${rand}-observe-health`;
-    const jsonPath = join(outDir, `${id}.json`);
+    migrateLegacyReportFiles(outDir, 'observe-health');
+    const id = runFileSuffix();
+    const jsonPath = reportFilePath(outDir, id);
     writeFileSync(jsonPath, JSON.stringify(report, null, 2));
     indexObserveWrite(report, jsonPath, outDir, id);
     return { id, jsonPath };