oh-my-knowledge 0.41.0 → 0.43.0

package/dist/eval-workflows/evaluation-pipeline/run-state.d.ts

@@ -22,7 +22,7 @@ export interface EvaluationRunState {
     runningJob: EvaluationJob;
     resolvedJobStore: JobStore | null;
 }
-export declare function initializeEvaluationRunState({ samplesPath, skillDir, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, concurrency, timeoutMs, noCache, project, owner, tags, runId, jobStore, persistJob, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }: {
+export declare function initializeEvaluationRunState({ samplesPath, skillDir, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, concurrency, timeoutMs, noCache, project, owner, tags, runId, jobStore, persistJob, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }: {
     samplesPath: string;
     skillDir: string;
     artifacts: Artifact[];
@@ -41,6 +41,7 @@ export declare function initializeEvaluationRunState({ samplesPath, skillDir, ar
     jobStore?: JobStore | null;
     persistJob?: boolean;
     repeat?: number;
+    holdoutRatio?: number;
     batch?: boolean;
     judgeRepeat?: number;
     judgeModels?: import('../../types/index.js').JudgeConfig[];

package/dist/eval-workflows/evaluation-pipeline/run-state.js

@@ -14,7 +14,7 @@
 import { buildEvaluationRequest, createFailedJob, createEvaluationRun, createQueuedJob, createSucceededJob, finalizeEvaluationRun, markJobRunning, failEvaluationRun, } from '../../eval-core/evaluation-job.js';
 import { createFileJobStore } from '../../server/job-store.js';
 import { DEFAULT_JOBS_DIR } from '../../eval-core/default-dirs.js';
-export async function initializeEvaluationRunState({ samplesPath, skillDir, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, concurrency, timeoutMs, noCache, project, owner, tags, runId, jobStore, persistJob, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }) {
+export async function initializeEvaluationRunState({ samplesPath, skillDir, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, concurrency, timeoutMs, noCache, project, owner, tags, runId, jobStore, persistJob, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }) {
     const effectiveJudges = judgeModels && judgeModels.length > 0
         ? judgeModels
         : [{ executor: judgeExecutorName, model: judgeModel }];
@@ -33,6 +33,7 @@ export async function initializeEvaluationRunState({ samplesPath, skillDir, arti
         owner,
         tags,
         repeat,
+        holdoutRatio,
         batch,
         judgeRepeat,
         judgeModels: effectiveJudges,

package/dist/eval-workflows/evaluation-pipeline.d.ts

@@ -61,6 +61,8 @@ export interface EvaluationPipelineOptions {
     layeredStats?: boolean;
     /** 透传到 meta.request.repeat */
     repeat?: number;
+    /** 透传到 meta.request.holdoutRatio；> 0 时 report-finalize 算 train/holdout 子集综合分。 */
+    holdoutRatio?: number;
     /** 透传到 meta.request.batch */
     batch?: boolean;
     /** 透传到 meta.request.judgeRepeat 与 grade()，每条 sample × dimension judge N 次 */
@@ -89,7 +91,7 @@ export interface EvaluationPipelineOptions {
     noDiagnostic?: boolean;
 }
 type VariantResult = import('../types/index.js').VariantResult;
-export declare function executeEvaluationPipeline({ samplesPath, samplesBaseDir, samplesSourceFiles, skillDir, samples, tasks, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, executor, judgeExecutor, outputDir, project, owner, tags, concurrency, timeoutMs, noCache, jobStore, persistJob, onProgress, skipConnectivity, verbose, retry, existingResults, requires: _requires, layeredStats, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, runId, lang, effort, noDiagnostic, }: EvaluationPipelineOptions): Promise<{
+export declare function executeEvaluationPipeline({ samplesPath, samplesBaseDir, samplesSourceFiles, skillDir, samples, tasks, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, executor, judgeExecutor, outputDir, project, owner, tags, concurrency, timeoutMs, noCache, jobStore, persistJob, onProgress, skipConnectivity, verbose, retry, existingResults, requires: _requires, layeredStats, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, runId, lang, effort, noDiagnostic, }: EvaluationPipelineOptions): Promise<{
     report: Report;
     filePath: string | null;
 }>;

package/dist/eval-workflows/evaluation-pipeline.js

@@ -33,7 +33,7 @@ export { _computeTestSetHashForTest } from './evaluation-pipeline/test-set-hash.
 export async function executeEvaluationPipeline({ samplesPath, samplesBaseDir, samplesSourceFiles, skillDir, samples, tasks, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, executor, judgeExecutor, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, concurrency = 1, timeoutMs, noCache = false, jobStore = null, persistJob = true, onProgress = null, skipConnectivity = false, verbose = false, retry = 0, existingResults, 
 // requires 现在由 runEvaluation 上游传给 doctor 处理; eval-pipeline 不再用
 // 但保留接口字段,避免破 programmatic API (类型层面接收, 内部忽略)
-requires: _requires, layeredStats = false, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias = true, budget, strictBaseline, runId, lang = 'zh', effort, noDiagnostic, }) {
+requires: _requires, layeredStats = false, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias = true, budget, strictBaseline, runId, lang = 'zh', effort, noDiagnostic, }) {
     const variantNames = artifacts.map((artifact) => artifact.name);
     const runState = await initializeEvaluationRunState({
         samplesPath,
@@ -54,6 +54,7 @@ requires: _requires, layeredStats = false, repeat, batch, judgeRepeat, judgeMode
         jobStore,
         persistJob,
         repeat,
+        holdoutRatio,
         batch,
         judgeRepeat,
         judgeModels,

package/dist/eval-workflows/run-evaluation.d.ts

@@ -32,6 +32,9 @@ interface CommonEvaluationOptions {
     /** --repeat N. 1 表示单次(默认); > 1 时在 runMultiple 层聚合 variance。
      *  记入 report.meta.request.repeat 让 meta 如实反映用户输入。 */
     repeat?: number;
+    /** --holdout-ratio R. 0 / 缺省 = 不切分(默认)。> 0 时 report-finalize 算 train/holdout
+     *  子集综合分(report.analysis.holdout),供 verdict 过拟合门控读取。 */
+    holdoutRatio?: number;
     /** --batch 模式标记, true 表示当前评测是 skill batch 流程。
      *  记入 report.meta.request.batch。 */
     batch?: boolean;
@@ -117,7 +120,7 @@ export interface DryRunReport extends DryRunBase {
     samplesPath: string;
     tasks: DryRunTask[];
 }
-export declare function runEvaluation({ samplesPath, skillDir, variantSpecs, model, outputDir, project, owner, tags, noJudge, dryRun, concurrency, timeoutMs, noCache, executorName, jobStore, persistJob, onProgress, skipConnectivity, skipDoctor, lang, mcpConfig, verbose, retry, resume, runId, layeredStats, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, effort, noDiagnostic, }: RunEvaluationOptions): Promise<{
+export declare function runEvaluation({ samplesPath, skillDir, variantSpecs, model, outputDir, project, owner, tags, noJudge, dryRun, concurrency, timeoutMs, noCache, executorName, jobStore, persistJob, onProgress, skipConnectivity, skipDoctor, lang, mcpConfig, verbose, retry, resume, runId, layeredStats, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, effort, noDiagnostic, }: RunEvaluationOptions): Promise<{
     report: Report | DryRunReport;
     filePath: string | null;
 }>;
@@ -133,7 +136,7 @@ export interface DryRunBatchReport extends DryRunBase {
     artifacts: DryRunBatchSkill[];
 }
 export declare function buildVarianceData(runs: Report[], bootstrapSamples?: number, seed?: number): VarianceData | null;
-export declare function runBatchEvaluation({ skillDir, model, outputDir, project, owner, tags, noJudge, dryRun, concurrency, timeoutMs, executorName, jobStore, persistJob, onProgress, onSkillProgress, skipConnectivity, skipDoctor, lang, mcpConfig, verbose, repeat, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, noCache, strictBaseline, variantAllowedSkills, }: RunBatchEvaluationOptions): Promise<{
+export declare function runBatchEvaluation({ skillDir, model, outputDir, project, owner, tags, noJudge, dryRun, concurrency, timeoutMs, executorName, jobStore, persistJob, onProgress, onSkillProgress, skipConnectivity, skipDoctor, lang, mcpConfig, verbose, repeat, holdoutRatio, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, noCache, strictBaseline, variantAllowedSkills, }: RunBatchEvaluationOptions): Promise<{
     report: BatchEvaluationReport | DryRunBatchReport;
     filePath: string | null;
 }>;

package/dist/eval-workflows/run-evaluation.js

@@ -8,7 +8,7 @@ import { buildDryRunTaskReport, prepareEvaluationRun, } from './evaluation-prepa
 import { executeEvaluationPipeline } from './evaluation-pipeline.js';
 import { findSaturationPoint } from '../analysis/saturation.js';
 import { bootstrapMeanCI, DEFAULT_BOOTSTRAP_ALPHA, DEFAULT_BOOTSTRAP_SAMPLES } from '../eval-core/bootstrap.js';
-export async function runEvaluation({ samplesPath, skillDir, variantSpecs = [], model = DEFAULT_MODEL, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, noJudge = false, dryRun = false, concurrency = 1, timeoutMs, noCache = false, executorName = 'claude', jobStore = null, persistJob = true, onProgress = null, skipConnectivity = false, skipDoctor = false, lang = 'zh', mcpConfig, verbose = false, retry = 0, resume, runId, layeredStats = false, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, effort, noDiagnostic, }) {
+export async function runEvaluation({ samplesPath, skillDir, variantSpecs = [], model = DEFAULT_MODEL, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, noJudge = false, dryRun = false, concurrency = 1, timeoutMs, noCache = false, executorName = 'claude', jobStore = null, persistJob = true, onProgress = null, skipConnectivity = false, skipDoctor = false, lang = 'zh', mcpConfig, verbose = false, retry = 0, resume, runId, layeredStats = false, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, effort, noDiagnostic, }) {
     // Unified judgeModels → derive single-judge fields for downstream pipeline / grading
     // (which still operate on string `judgeModel` + `judgeExecutorName` fields per call).
     const effectiveJudgeModels = judgeModels && judgeModels.length > 0
@@ -165,6 +165,7 @@ export async function runEvaluation({ samplesPath, skillDir, variantSpecs = [],
         requires,
         layeredStats,
         repeat,
+        holdoutRatio,
         batch,
         judgeRepeat,
         judgeModels,
@@ -366,7 +367,7 @@ export function buildVarianceData(runs, bootstrapSamples = DEFAULT_BOOTSTRAP_SAM
         ...(saturation ? { saturation } : {}),
     };
 }
-export async function runBatchEvaluation({ skillDir, model = DEFAULT_MODEL, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, noJudge = false, dryRun = false, concurrency = 1, timeoutMs, executorName = 'claude', jobStore = null, persistJob = true, onProgress = null, onSkillProgress = null, skipConnectivity = false, skipDoctor = false, lang = 'zh', mcpConfig, verbose = false, repeat, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, noCache = false, strictBaseline, variantAllowedSkills, }) {
+export async function runBatchEvaluation({ skillDir, model = DEFAULT_MODEL, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, noJudge = false, dryRun = false, concurrency = 1, timeoutMs, executorName = 'claude', jobStore = null, persistJob = true, onProgress = null, onSkillProgress = null, skipConnectivity = false, skipDoctor = false, lang = 'zh', mcpConfig, verbose = false, repeat, holdoutRatio, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, noCache = false, strictBaseline, variantAllowedSkills, }) {
     // Same unified judge derivation as runEvaluation (downstream pipeline / report build
     // still uses single judgeModel + judgeExecutorName per call).
     const effectiveJudgeModels = judgeModels && judgeModels.length > 0
@@ -485,12 +486,14 @@ export async function runBatchEvaluation({ skillDir, model = DEFAULT_MODEL, outp
         strictBaseline,
         variantAllowedSkills,
         runSingleEvaluation: async (options) => {
-            // repeat > 1 时走 runMultiple 做 variance; batch=true 标记让 meta.request 如实反映
+            // repeat > 1 时走 runMultiple 做 variance; batch=true 标记让 meta.request 如实反映。
+            // holdoutRatio 显式转发(executeBatchEvaluationRuns 不一定把它塞进 options),否则
+            // batch 子报告 meta.request.holdoutRatio 丢失、过拟合门控对 batch 静默失效。
             if (repeat && repeat > 1) {
-                const multi = await runMultiple({ ...options, repeat, batch: true, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget });
+                const multi = await runMultiple({ ...options, repeat, holdoutRatio, batch: true, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget });
                 return { report: multi.report, filePath: multi.filePath };
             }
-            const result = await runEvaluation({ ...options, batch: true, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget });
+            const result = await runEvaluation({ ...options, batch: true, holdoutRatio, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget });
             return { report: result.report, filePath: result.filePath };
         },
     });

package/dist/inputs/eval-config.js

@@ -171,6 +171,11 @@ function validateEvalConfig(parsed, configPath) {
     };
     assertPositiveIntOpt('repeat');
     assertPositiveIntOpt('judgeRepeat');
+    if (obj.holdoutRatio !== undefined) {
+        if (typeof obj.holdoutRatio !== 'number' || !Number.isFinite(obj.holdoutRatio) || obj.holdoutRatio <= 0 || obj.holdoutRatio >= 1) {
+            throw new Error(`${configPath}: holdoutRatio must be a number in (0, 1)`);
+        }
+    }
     if (obj.bootstrapSamples !== undefined) {
         if (typeof obj.bootstrapSamples !== 'number' || !Number.isFinite(obj.bootstrapSamples) || obj.bootstrapSamples < 100) {
             throw new Error(`${configPath}: bootstrapSamples must be a number ≥ 100`);
@@ -256,6 +261,7 @@ function validateEvalConfig(parsed, configPath) {
         variants,
         budget,
         repeat: obj.repeat,
+        holdoutRatio: obj.holdoutRatio,
         judgeRepeat: obj.judgeRepeat,
         bootstrap: obj.bootstrap,
         bootstrapSamples: obj.bootstrapSamples,

package/dist/inputs/sample-locator.d.ts

@@ -0,0 +1,23 @@
+export declare const PROJECT_SAMPLE_FILENAMES: readonly ["eval-samples.json", "eval-samples.yaml", "eval-samples.yml"];
+export declare const SKILL_LOCAL_SAMPLE_FILENAMES: readonly ["samples.json", "samples.yaml", "samples.yml"];
+export interface DeprecatedSkillSamplesHint {
+    oldPath: string;
+    newPath: string;
+}
+export declare function isExistingDirectory(path: string): boolean;
+export declare function hasLoadableSampleFile(dir: string): boolean;
+export declare function findProjectSamplesFile(dir: string): string | null;
+export declare function skillLocalSamplesDir(skillRoot: string): string;
+export declare function defaultSkillLocalSamplesFile(skillRoot: string): string;
+export declare function hasUsableSamplesPath(path: string): boolean;
+export declare function findSkillLocalSamplesDir(skillRoot: string): string | null;
+export declare function findSkillSamplesPath(skillRoot: string): string | null;
+export declare function findDeprecatedSkillSamplesHint(skillRoot: string): DeprecatedSkillSamplesHint | null;
+export declare function findFlatSkillSamplesPath(skillDir: string, skillName: string): string | null;
+export declare function defaultFlatSkillSamplesFile(skillDir: string, skillName: string): string;
+export declare function isDirectorySkillRoot(path: string): boolean;
+export declare function findNamedSkillSamplesPath(skillDir: string, skillName: string): string | null;
+export declare function findSingleTreatmentSamplesPath(treatmentExpr: string, skillDir: string, cwd?: string): string | null;
+export declare function findSingleTreatmentDeprecatedSamplesHint(treatmentExpr: string, skillDir: string, cwd?: string): DeprecatedSkillSamplesHint | null;
+export declare function findDoctorSamplesPath(target: string | null, cwd: string): string | null;
+export declare function findDoctorDeprecatedSamplesHint(target: string | null, cwd: string): DeprecatedSkillSamplesHint | null;

package/dist/inputs/sample-locator.js

@@ -0,0 +1,195 @@
+import { existsSync, readdirSync, statSync } from 'node:fs';
+import { basename, dirname, join, resolve } from 'node:path';
+export const PROJECT_SAMPLE_FILENAMES = ['eval-samples.json', 'eval-samples.yaml', 'eval-samples.yml'];
+export const SKILL_LOCAL_SAMPLE_FILENAMES = ['samples.json', 'samples.yaml', 'samples.yml'];
+const SAMPLE_FILE_RE = /\.(json|ya?ml)$/i;
+const RESERVED_SAMPLE_FILE_RE = /^(report|health|_)/i;
+function isExistingFile(path) {
+    try {
+        return existsSync(path) && statSync(path).isFile();
+    }
+    catch {
+        return false;
+    }
+}
+export function isExistingDirectory(path) {
+    try {
+        return existsSync(path) && statSync(path).isDirectory();
+    }
+    catch {
+        return false;
+    }
+}
+export function hasLoadableSampleFile(dir) {
+    if (!isExistingDirectory(dir))
+        return false;
+    try {
+        return readdirSync(dir).some((file) => SAMPLE_FILE_RE.test(file) && !RESERVED_SAMPLE_FILE_RE.test(file));
+    }
+    catch {
+        return false;
+    }
+}
+export function findProjectSamplesFile(dir) {
+    for (const name of PROJECT_SAMPLE_FILENAMES) {
+        const candidate = join(dir, name);
+        if (isExistingFile(candidate))
+            return candidate;
+    }
+    return null;
+}
+export function skillLocalSamplesDir(skillRoot) {
+    return join(skillRoot, '.omk');
+}
+export function defaultSkillLocalSamplesFile(skillRoot) {
+    return join(skillLocalSamplesDir(skillRoot), SKILL_LOCAL_SAMPLE_FILENAMES[0]);
+}
+export function hasUsableSamplesPath(path) {
+    if (isExistingFile(path))
+        return true;
+    return hasLoadableSampleFile(path);
+}
+export function findSkillLocalSamplesDir(skillRoot) {
+    const dir = skillLocalSamplesDir(skillRoot);
+    return hasLoadableSampleFile(dir) ? dir : null;
+}
+export function findSkillSamplesPath(skillRoot) {
+    return findSkillLocalSamplesDir(skillRoot);
+}
+export function findDeprecatedSkillSamplesHint(skillRoot) {
+    if (!isDirectorySkillRoot(skillRoot))
+        return null;
+    for (const name of PROJECT_SAMPLE_FILENAMES) {
+        const oldPath = join(skillRoot, name);
+        if (isExistingFile(oldPath)) {
+            return { oldPath, newPath: defaultSkillLocalSamplesFile(skillRoot) };
+        }
+    }
+    return null;
+}
+export function findFlatSkillSamplesPath(skillDir, skillName) {
+    for (const ext of ['json', 'yaml', 'yml']) {
+        const candidate = join(skillDir, `${skillName}.eval-samples.${ext}`);
+        if (isExistingFile(candidate))
+            return candidate;
+    }
+    return null;
+}
+export function defaultFlatSkillSamplesFile(skillDir, skillName) {
+    return join(skillDir, `${skillName}.eval-samples.json`);
+}
+export function isDirectorySkillRoot(path) {
+    return isExistingDirectory(path) && isExistingFile(join(path, 'SKILL.md'));
+}
+export function findNamedSkillSamplesPath(skillDir, skillName) {
+    const flatSkillPath = join(skillDir, `${skillName}.md`);
+    if (isExistingFile(flatSkillPath)) {
+        return findFlatSkillSamplesPath(skillDir, skillName);
+    }
+    const dirSkillRoot = join(skillDir, skillName);
+    if (isDirectorySkillRoot(dirSkillRoot)) {
+        return findSkillSamplesPath(dirSkillRoot);
+    }
+    return null;
+}
+function findSamplesForExistingSkillPath(path) {
+    if (isExistingDirectory(path)) {
+        return findSkillSamplesPath(path);
+    }
+    const parent = dirname(path);
+    if (basename(path) === 'SKILL.md') {
+        return findSkillSamplesPath(parent);
+    }
+    if (/\.md$/i.test(path)) {
+        const skillName = basename(path).replace(/\.md$/i, '');
+        return findFlatSkillSamplesPath(parent, skillName);
+    }
+    return findProjectSamplesFile(parent);
+}
+function findDeprecatedSamplesForExistingSkillPath(path) {
+    if (isExistingDirectory(path)) {
+        return findDeprecatedSkillSamplesHint(path);
+    }
+    if (basename(path) === 'SKILL.md') {
+        return findDeprecatedSkillSamplesHint(dirname(path));
+    }
+    return null;
+}
+export function findSingleTreatmentSamplesPath(treatmentExpr, skillDir, cwd = process.cwd()) {
+    const resolved = resolve(cwd, treatmentExpr);
+    if (existsSync(resolved)) {
+        const samplesPath = findSamplesForExistingSkillPath(resolved);
+        if (samplesPath)
+            return samplesPath;
+    }
+    return findNamedSkillSamplesPath(skillDir, treatmentExpr);
+}
+export function findSingleTreatmentDeprecatedSamplesHint(treatmentExpr, skillDir, cwd = process.cwd()) {
+    const resolved = resolve(cwd, treatmentExpr);
+    if (existsSync(resolved)) {
+        return findDeprecatedSamplesForExistingSkillPath(resolved);
+    }
+    const flatSkillPath = join(skillDir, `${treatmentExpr}.md`);
+    if (isExistingFile(flatSkillPath))
+        return null;
+    return findDeprecatedSkillSamplesHint(join(skillDir, treatmentExpr));
+}
+function projectSampleSearchDirs(target, cwd) {
+    const dirs = [];
+    const add = (dir) => {
+        const abs = resolve(dir);
+        if (!dirs.includes(abs))
+            dirs.push(abs);
+    };
+    if (target) {
+        const absTarget = resolve(target);
+        if (existsSync(absTarget)) {
+            if (isExistingDirectory(absTarget)) {
+                if (isDirectorySkillRoot(absTarget)) {
+                    add(dirname(dirname(absTarget)));
+                }
+                else {
+                    add(absTarget);
+                    add(dirname(absTarget));
+                    add(dirname(dirname(absTarget)));
+                }
+            }
+            else {
+                const parent = dirname(absTarget);
+                if (basename(absTarget) === 'SKILL.md') {
+                    add(dirname(dirname(parent)));
+                }
+                else {
+                    add(parent);
+                    add(dirname(parent));
+                }
+            }
+        }
+    }
+    add(cwd);
+    return dirs;
+}
+export function findDoctorSamplesPath(target, cwd) {
+    if (target) {
+        const absTarget = resolve(target);
+        if (existsSync(absTarget)) {
+            const targetSamples = findSamplesForExistingSkillPath(absTarget);
+            if (targetSamples)
+                return targetSamples;
+        }
+    }
+    for (const dir of projectSampleSearchDirs(target, cwd)) {
+        const samplesPath = findProjectSamplesFile(dir);
+        if (samplesPath)
+            return samplesPath;
+    }
+    return null;
+}
+export function findDoctorDeprecatedSamplesHint(target, cwd) {
+    if (!target)
+        return null;
+    const absTarget = resolve(cwd, target);
+    if (!existsSync(absTarget))
+        return null;
+    return findDeprecatedSamplesForExistingSkillPath(absTarget);
+}

package/dist/inputs/skill-loader.js

@@ -5,6 +5,7 @@ import { execFileSync } from 'node:child_process';
 import { extractSkillHardRules, extractSkillWorkflows } from '../shared/hard-rules.js';
 import { hashArtifactSource, hashBytes, isDistributablePath } from './content-hash.js';
 import { materializeIsolatedCopy } from './materialize-copy.js';
+import { findFlatSkillSamplesPath, findSkillSamplesPath } from './sample-locator.js';
 function parseFrontmatterPreflight(content) {
     const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---/);
     if (!match)
@@ -388,14 +389,9 @@ export function discoverBatchSkills(skillDir) {
         const mdMatch = entry.endsWith('.md') && !entry.endsWith('.eval-samples.json');
         if (mdMatch) {
             const name = entry.slice(0, -3);
-            const candidates = [
-                join(skillDir, name, '.omk'),
-                join(skillDir, `${name}.eval-samples.json`),
-                join(skillDir, `${name}.eval-samples.yaml`),
-                join(skillDir, `${name}.eval-samples.yml`),
-            ].filter(existsSync);
-            if (candidates.length > 0) {
-                skills.push({ name, skillPath: join(skillDir, entry), samplesPath: candidates[0] });
+            const samplesPath = findFlatSkillSamplesPath(skillDir, name);
+            if (samplesPath) {
+                skills.push({ name, skillPath: join(skillDir, entry), samplesPath });
             }
             else {
                 warned.push(name);
@@ -406,15 +402,9 @@ export function discoverBatchSkills(skillDir) {
             const skillMd = join(entryPath, 'SKILL.md');
             if (!existsSync(skillMd))
                 continue;
-            // .omk/ dir (loadSamples handles dir mode) > .omk/samples.json > eval-samples.{json,yaml,yml}
-            const omkDir = join(entryPath, '.omk');
-            const candidates = [
-                ...(existsSync(omkDir) ? [omkDir] : []),
-                join(entryPath, 'eval-samples.json'),
-                join(entryPath, 'eval-samples.yaml'),
-                join(entryPath, 'eval-samples.yml'),
-            ];
-            const samplesPath = candidates.find(existsSync);
+            if (existsSync(join(skillDir, `${entry}.md`)))
+                continue;
+            const samplesPath = findSkillSamplesPath(entryPath);
             if (samplesPath) {
                 skills.push({ name: entry, skillPath: skillMd, samplesPath });
             }

package/dist/observability/inbox.js

@@ -2,13 +2,15 @@ import { createHash } from 'node:crypto';
 import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { OMK_HOME } from '../eval-core/default-dirs.js';
+import { isReportFileName, reportFilePath } from '../eval-core/artifact-file-names.js';
+import { migrateLegacyReportFiles } from '../eval-core/report-file-migration.js';
 import { extractGapSignalsFromTrace } from '../analysis/gap-analyzer.js';
 import { ccTracesToResultEntries } from './trace-adapter.js';
 import { isSearchToolCall, toolCallQuery } from '../shared/tool-search.js';
 import { durationMsBetween } from '../shared/time.js';
 import { buildObservationExperienceReport, normalizeObservationExperienceReport, } from './experience.js';
 // observe inbox（观测收件箱）产物根目录。导出名沿用 *_OBSERVATIONS_DIR 以少动 importer,
-// 但落盘目录已统一到 observe-inbox 词根(命令 omk observe inbox / 文件 *-observe-inbox.json / kind observe-inbox)。
+// 但落盘目录已统一到 observe-inbox 词根(命令 omk observe inbox / kind observe-inbox)。
 // 项目级 .omk/observe-inbox 优先、全局兜底 —— 这套 project/global 归属是既有正常行为,本次只改名不改归属。
 export const DEFAULT_PROJECT_OBSERVATIONS_DIR = join(process.cwd(), '.omk', 'observe-inbox');
 export const DEFAULT_GLOBAL_OBSERVATIONS_DIR = join(OMK_HOME, 'observe-inbox');
@@ -529,10 +531,11 @@ function compareInboxItems(a, b) {
 }
 export function saveObservationInboxReport(report, outDir = DEFAULT_OBSERVATIONS_DIR) {
     mkdirSync(outDir, { recursive: true });
+    migrateLegacyReportFiles(outDir, 'observe-inbox');
     // 保留毫秒;同秒不同毫秒生成的两份 report 不应静默互相覆盖。
     // 例: '2026-05-07T12:00:00.999Z' → '2026-05-07T12-00-00-999'
     const stamp = report.meta.generatedAt.replace(/[:.]/g, '-').replace(/Z$/, '');
-    const path = join(outDir, `${stamp}-observe-inbox.json`);
+    const path = reportFilePath(outDir, stamp);
     writeFileSync(path, JSON.stringify(report, null, 2));
     return path;
 }
@@ -543,8 +546,9 @@ export function loadObservationInboxReports(dir = DEFAULT_OBSERVATIONS_DIR) {
         }
         return [];
     }
+    migrateLegacyReportFiles(dir, 'observe-inbox');
     return readdirSync(dir)
-        .filter((file) => file.endsWith('-observe-inbox.json'))
+        .filter(isReportFileName)
         .map((file) => {
         try {
             const report = normalizeObservationInboxReport(JSON.parse(readFileSync(join(dir, file), 'utf-8')));

package/dist/renderer/summary.js

@@ -101,6 +101,30 @@ function computeMedianCVPercent(report) {
     const stab = medianStabilityCV(report);
     return stab ? stab.cv * 100 : null;
 }
+// Verdict caveats（过拟合 / 知识缺口）渲染进 pill —— 让 HTML 报告和 CLI 说同一件事:
+// CLI 在 verbose rationale 里给这两条,HTML 之前只剩一个泛化后的 level、看不到触发原因。
+// 用 result.caveats 的结构化数据 i18n,而不是重解析 zh rationale 串。
+function renderVerdictCaveats(caveats, lang) {
+    if (!caveats)
+        return '';
+    const lines = [];
+    if (caveats.overfitting) {
+        const c = caveats.overfitting;
+        lines.push(lang === 'zh'
+            ? `⚠ 过拟合敞口:${c.variant} 训练 ${c.trainScore.toFixed(2)} / 留出 ${c.holdoutScore.toFixed(2)}(差 ${c.gap.toFixed(2)}),提升可能不泛化`
+            : `⚠ Overfitting: ${c.variant} train ${c.trainScore.toFixed(2)} / holdout ${c.holdoutScore.toFixed(2)} (gap ${c.gap.toFixed(2)}) — gain may not generalize`);
+    }
+    if (caveats.gapSignal) {
+        const g = caveats.gapSignal;
+        const wm = g.testSetHash ? g.testSetHash.slice(0, 8) : (g.testSetPath ?? '');
+        lines.push(lang === 'zh'
+            ? `知识缺口率 ${g.gapRatePct}%(test set ${wm},informational)`
+            : `Knowledge gap ${g.gapRatePct}% (test set ${wm}, informational)`);
+    }
+    if (lines.length === 0)
+        return '';
+    return `<div class="page-verdict-caveats">${lines.map((l) => `<span class="page-verdict-caveat">${e(l)}</span>`).join('')}</div>`;
+}
 export function renderVerdictPill(report, lang) {
     let result;
     try {
@@ -110,8 +134,16 @@ export function renderVerdictPill(report, lang) {
         return '';
     }
     const level = result.level;
-    const pair = result.perPair?.[0];
+    // representative = top-level worst pair（与 CLI 同口径）,不是第一对。多 treatment 报告里
+    // worst pair 不一定是 perPair[0],用它才不会把错的 treatment 名写进结论。fallback 兼容旧路径。
+    const pair = result.representative ?? result.perPair?.[0];
     const oneLine = verdictOneLine(level, lang, pair?.treatment, pair?.control);
+    // Δ/CI 证据必须跟文案指同一对:按 representative 匹配对应的 pairComparison(alpha 也走这对),
+    // 否则多 treatment 报告会出现「文案 t2、数字 t1」的混搭。匹配不到 / 无 representative 时 fallback [0]。
+    const pairComparisons = report.meta?.pairComparisons;
+    const activeComparison = (pair
+        ? pairComparisons?.find((p) => p.treatment === pair.treatment && p.control === pair.control)
+        : undefined) ?? pairComparisons?.[0];
     const tooltip = levelTooltip(level, lang);
     const prefix = lang === 'zh' ? '测评结论' : 'Verdict';
     // 机器可读 enum 永远是 level token; 显示给用户的文字按 lang i18n.
@@ -120,7 +152,7 @@ export function renderVerdictPill(report, lang) {
     // hero 只放「答案」: 分差是 verdict 的核心证据数字, 单独一枚 chip。
     // 评测规模 (用例数 × 轮次) 走「实验配置」section 的 subtitle 那条 canonical 路径,
     // 不在 hero 里重复; CV / CI 走 chip tooltip + 方法学审计 / 波动表。
-    const ci = report.meta?.pairComparisons?.[0]?.diffBootstrapCI;
+    const ci = activeComparison?.diffBootstrapCI;
     const cvPct = computeMedianCVPercent(report);
     const metrics = [];
     if (ci) {
@@ -128,7 +160,7 @@ export function renderVerdictPill(report, lang) {
         const cvSuffix = cvPct != null
             ? (lang === 'zh' ? `;多轮稳定性 CV=${cvPct.toFixed(1)}% (${cvPct < 5 ? '稳' : cvPct <= STABILITY_UNSTABLE_CV * 100 ? '中' : '不稳'})` : `; CV=${cvPct.toFixed(1)}% (${cvPct < 5 ? 'stable' : cvPct <= STABILITY_UNSTABLE_CV * 100 ? 'moderate' : 'unstable'})`)
             : '';
-        const pctLabel = ciLevelLabel(report.meta?.pairComparisons?.[0]?.alpha);
+        const pctLabel = ciLevelLabel(activeComparison?.alpha);
         const ciTipBase = lang === 'zh'
             ? `实验组与对照组综合分均值差(Δ)。bootstrap ${pctLabel} 可信区间 [${ci.low}, ${ci.high}]，${ci.significant ? '不含 0 = 差异显著' : '跨过 0 = 差异不显著'}${cvSuffix}`
             : `Treatment minus control mean composite score (Δ). Bootstrap ${pctLabel} CI [${ci.low}, ${ci.high}], ${ci.significant ? 'excludes 0 ⇒ significant' : 'spans 0 ⇒ not significant'}${cvSuffix}`;
@@ -146,6 +178,7 @@ export function renderVerdictPill(report, lang) {
       <span class="page-verdict-badge"><span class="page-verdict-badge-dot" aria-hidden="true">●</span>${e(levelDisplay)}</span>
       <span class="page-verdict-text">${e(oneLine)}</span>
     </div>
+    ${renderVerdictCaveats(result.caveats, lang)}
     ${metricChips ? `<div class="page-verdict-metrics">${metricChips}</div>` : ''}
   </section>`;
 }

package/dist/server/report-server.js

@@ -14,6 +14,8 @@ import { renderManagedList, renderManagedHistory } from '../renderer/managed-his
 import { DEFAULT_JOBS_DIR } from '../eval-core/default-dirs.js';
 import { resolveObserveHealthDir, projectObserveHealthDir, resolveDoctorsDir, projectDoctorsDir, projectReportsDir, globalReportsDir } from '../eval-core/measurement-dirs.js';
 import { listObserveCards, listDoctorCards, listLiveObserveCards } from '../eval-core/artifact-index.js';
+import { isReportFileName, reportFilePath, reportFileStem } from '../eval-core/artifact-file-names.js';
+import { migrateLegacyReportFiles } from '../eval-core/report-file-migration.js';
 import { buildSkillIndex } from './skill-index.js';
 import { createFileJobStore } from './job-store.js';
 import { createFileStore, queryJob, queryJobList, queryRun, queryRunList, queryTrend } from './report-store.js';
@@ -46,18 +48,20 @@ function loadChartJsBundle() {
 }
 function listAnalyses(dir, includeCards = false) {
     const items = [];
+    migrateLegacyReportFiles(dir, 'observe-health');
     // live 扫描 dir 存在才做;dir 不存在(默认机器级模式下当前项目还没 .omk/observe-health、全局也空)时 live 为空,
     // 但**不能早退** —— 后面仍要按 includeCards 合并别项目卡片,否则 observe 列表会与合卡片的 /api/skills 口径分裂。
     if (existsSync(dir)) {
         for (const file of readdirSync(dir)) {
-            if (!file.endsWith('.json'))
+            const id = reportFileStem(file);
+            if (!id)
                 continue;
             try {
                 const data = JSON.parse(readFileSync(join(dir, file), 'utf-8'));
                 if (!data.meta || !data.overall)
                     continue;
                 items.push({
-                    id: file.replace(/\.json$/, ''),
+                    id,
                     generatedAt: data.meta.generatedAt,
                     sessionCount: data.meta.sessionCount,
                     segmentCount: data.meta.segmentCount,
@@ -89,7 +93,8 @@ function listAnalyses(dir, includeCards = false) {
     return items;
 }
 function loadAnalysis(dir, id, includeCards = false) {
-    const path = join(dir, `${id}.json`);
+    migrateLegacyReportFiles(dir, 'observe-health');
+    const path = reportFilePath(dir, id);
     if (existsSync(path)) {
         try {
             return JSON.parse(readFileSync(path, 'utf-8'));
@@ -114,9 +119,10 @@ function loadAnalysis(dir, id, includeCards = false) {
  *  优先返回含该 skill 的那份；都不含时回退首个 id 命中（单 skill / 无参行为不变）。 */
 function loadDoctorReport(dir, id, skillName, includeCards = false) {
     let fallback = null;
+    migrateLegacyReportFiles(dir, 'doctor');
     if (existsSync(dir)) {
         for (const file of readdirSync(dir)) {
-            if (!file.endsWith('.json'))
+            if (!isReportFileName(file))
                 continue;
             try {
                 const data = JSON.parse(readFileSync(join(dir, file), 'utf-8'));