oh-my-knowledge 0.41.0 → 0.42.0

package/dist/eval-workflows/evaluation-pipeline/run-state.d.ts

@@ -22,7 +22,7 @@ export interface EvaluationRunState {
     runningJob: EvaluationJob;
     resolvedJobStore: JobStore | null;
 }
-export declare function initializeEvaluationRunState({ samplesPath, skillDir, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, concurrency, timeoutMs, noCache, project, owner, tags, runId, jobStore, persistJob, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }: {
+export declare function initializeEvaluationRunState({ samplesPath, skillDir, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, concurrency, timeoutMs, noCache, project, owner, tags, runId, jobStore, persistJob, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }: {
     samplesPath: string;
     skillDir: string;
     artifacts: Artifact[];
@@ -41,6 +41,7 @@ export declare function initializeEvaluationRunState({ samplesPath, skillDir, ar
     jobStore?: JobStore | null;
     persistJob?: boolean;
     repeat?: number;
+    holdoutRatio?: number;
     batch?: boolean;
     judgeRepeat?: number;
     judgeModels?: import('../../types/index.js').JudgeConfig[];

package/dist/eval-workflows/evaluation-pipeline/run-state.js

@@ -14,7 +14,7 @@
 import { buildEvaluationRequest, createFailedJob, createEvaluationRun, createQueuedJob, createSucceededJob, finalizeEvaluationRun, markJobRunning, failEvaluationRun, } from '../../eval-core/evaluation-job.js';
 import { createFileJobStore } from '../../server/job-store.js';
 import { DEFAULT_JOBS_DIR } from '../../eval-core/default-dirs.js';
-export async function initializeEvaluationRunState({ samplesPath, skillDir, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, concurrency, timeoutMs, noCache, project, owner, tags, runId, jobStore, persistJob, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }) {
+export async function initializeEvaluationRunState({ samplesPath, skillDir, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, concurrency, timeoutMs, noCache, project, owner, tags, runId, jobStore, persistJob, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }) {
     const effectiveJudges = judgeModels && judgeModels.length > 0
         ? judgeModels
         : [{ executor: judgeExecutorName, model: judgeModel }];
@@ -33,6 +33,7 @@ export async function initializeEvaluationRunState({ samplesPath, skillDir, arti
         owner,
         tags,
         repeat,
+        holdoutRatio,
         batch,
         judgeRepeat,
         judgeModels: effectiveJudges,

package/dist/eval-workflows/evaluation-pipeline.d.ts

@@ -61,6 +61,8 @@ export interface EvaluationPipelineOptions {
     layeredStats?: boolean;
     /** 透传到 meta.request.repeat */
     repeat?: number;
+    /** 透传到 meta.request.holdoutRatio；> 0 时 report-finalize 算 train/holdout 子集综合分。 */
+    holdoutRatio?: number;
     /** 透传到 meta.request.batch */
     batch?: boolean;
     /** 透传到 meta.request.judgeRepeat 与 grade()，每条 sample × dimension judge N 次 */
@@ -89,7 +91,7 @@ export interface EvaluationPipelineOptions {
     noDiagnostic?: boolean;
 }
 type VariantResult = import('../types/index.js').VariantResult;
-export declare function executeEvaluationPipeline({ samplesPath, samplesBaseDir, samplesSourceFiles, skillDir, samples, tasks, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, executor, judgeExecutor, outputDir, project, owner, tags, concurrency, timeoutMs, noCache, jobStore, persistJob, onProgress, skipConnectivity, verbose, retry, existingResults, requires: _requires, layeredStats, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, runId, lang, effort, noDiagnostic, }: EvaluationPipelineOptions): Promise<{
+export declare function executeEvaluationPipeline({ samplesPath, samplesBaseDir, samplesSourceFiles, skillDir, samples, tasks, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, executor, judgeExecutor, outputDir, project, owner, tags, concurrency, timeoutMs, noCache, jobStore, persistJob, onProgress, skipConnectivity, verbose, retry, existingResults, requires: _requires, layeredStats, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, runId, lang, effort, noDiagnostic, }: EvaluationPipelineOptions): Promise<{
     report: Report;
     filePath: string | null;
 }>;

package/dist/eval-workflows/evaluation-pipeline.js

@@ -33,7 +33,7 @@ export { _computeTestSetHashForTest } from './evaluation-pipeline/test-set-hash.
 export async function executeEvaluationPipeline({ samplesPath, samplesBaseDir, samplesSourceFiles, skillDir, samples, tasks, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, executor, judgeExecutor, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, concurrency = 1, timeoutMs, noCache = false, jobStore = null, persistJob = true, onProgress = null, skipConnectivity = false, verbose = false, retry = 0, existingResults, 
 // requires 现在由 runEvaluation 上游传给 doctor 处理; eval-pipeline 不再用
 // 但保留接口字段,避免破 programmatic API (类型层面接收, 内部忽略)
-requires: _requires, layeredStats = false, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias = true, budget, strictBaseline, runId, lang = 'zh', effort, noDiagnostic, }) {
+requires: _requires, layeredStats = false, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias = true, budget, strictBaseline, runId, lang = 'zh', effort, noDiagnostic, }) {
     const variantNames = artifacts.map((artifact) => artifact.name);
     const runState = await initializeEvaluationRunState({
         samplesPath,
@@ -54,6 +54,7 @@ requires: _requires, layeredStats = false, repeat, batch, judgeRepeat, judgeMode
         jobStore,
         persistJob,
         repeat,
+        holdoutRatio,
         batch,
         judgeRepeat,
         judgeModels,

package/dist/eval-workflows/run-evaluation.d.ts

@@ -32,6 +32,9 @@ interface CommonEvaluationOptions {
     /** --repeat N. 1 表示单次(默认); > 1 时在 runMultiple 层聚合 variance。
      *  记入 report.meta.request.repeat 让 meta 如实反映用户输入。 */
     repeat?: number;
+    /** --holdout-ratio R. 0 / 缺省 = 不切分(默认)。> 0 时 report-finalize 算 train/holdout
+     *  子集综合分(report.analysis.holdout),供 verdict 过拟合门控读取。 */
+    holdoutRatio?: number;
     /** --batch 模式标记, true 表示当前评测是 skill batch 流程。
      *  记入 report.meta.request.batch。 */
     batch?: boolean;
@@ -117,7 +120,7 @@ export interface DryRunReport extends DryRunBase {
     samplesPath: string;
     tasks: DryRunTask[];
 }
-export declare function runEvaluation({ samplesPath, skillDir, variantSpecs, model, outputDir, project, owner, tags, noJudge, dryRun, concurrency, timeoutMs, noCache, executorName, jobStore, persistJob, onProgress, skipConnectivity, skipDoctor, lang, mcpConfig, verbose, retry, resume, runId, layeredStats, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, effort, noDiagnostic, }: RunEvaluationOptions): Promise<{
+export declare function runEvaluation({ samplesPath, skillDir, variantSpecs, model, outputDir, project, owner, tags, noJudge, dryRun, concurrency, timeoutMs, noCache, executorName, jobStore, persistJob, onProgress, skipConnectivity, skipDoctor, lang, mcpConfig, verbose, retry, resume, runId, layeredStats, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, effort, noDiagnostic, }: RunEvaluationOptions): Promise<{
     report: Report | DryRunReport;
     filePath: string | null;
 }>;
@@ -133,7 +136,7 @@ export interface DryRunBatchReport extends DryRunBase {
     artifacts: DryRunBatchSkill[];
 }
 export declare function buildVarianceData(runs: Report[], bootstrapSamples?: number, seed?: number): VarianceData | null;
-export declare function runBatchEvaluation({ skillDir, model, outputDir, project, owner, tags, noJudge, dryRun, concurrency, timeoutMs, executorName, jobStore, persistJob, onProgress, onSkillProgress, skipConnectivity, skipDoctor, lang, mcpConfig, verbose, repeat, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, noCache, strictBaseline, variantAllowedSkills, }: RunBatchEvaluationOptions): Promise<{
+export declare function runBatchEvaluation({ skillDir, model, outputDir, project, owner, tags, noJudge, dryRun, concurrency, timeoutMs, executorName, jobStore, persistJob, onProgress, onSkillProgress, skipConnectivity, skipDoctor, lang, mcpConfig, verbose, repeat, holdoutRatio, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, noCache, strictBaseline, variantAllowedSkills, }: RunBatchEvaluationOptions): Promise<{
     report: BatchEvaluationReport | DryRunBatchReport;
     filePath: string | null;
 }>;

package/dist/eval-workflows/run-evaluation.js

@@ -8,7 +8,7 @@ import { buildDryRunTaskReport, prepareEvaluationRun, } from './evaluation-prepa
 import { executeEvaluationPipeline } from './evaluation-pipeline.js';
 import { findSaturationPoint } from '../analysis/saturation.js';
 import { bootstrapMeanCI, DEFAULT_BOOTSTRAP_ALPHA, DEFAULT_BOOTSTRAP_SAMPLES } from '../eval-core/bootstrap.js';
-export async function runEvaluation({ samplesPath, skillDir, variantSpecs = [], model = DEFAULT_MODEL, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, noJudge = false, dryRun = false, concurrency = 1, timeoutMs, noCache = false, executorName = 'claude', jobStore = null, persistJob = true, onProgress = null, skipConnectivity = false, skipDoctor = false, lang = 'zh', mcpConfig, verbose = false, retry = 0, resume, runId, layeredStats = false, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, effort, noDiagnostic, }) {
+export async function runEvaluation({ samplesPath, skillDir, variantSpecs = [], model = DEFAULT_MODEL, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, noJudge = false, dryRun = false, concurrency = 1, timeoutMs, noCache = false, executorName = 'claude', jobStore = null, persistJob = true, onProgress = null, skipConnectivity = false, skipDoctor = false, lang = 'zh', mcpConfig, verbose = false, retry = 0, resume, runId, layeredStats = false, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, effort, noDiagnostic, }) {
     // Unified judgeModels → derive single-judge fields for downstream pipeline / grading
     // (which still operate on string `judgeModel` + `judgeExecutorName` fields per call).
     const effectiveJudgeModels = judgeModels && judgeModels.length > 0
@@ -165,6 +165,7 @@ export async function runEvaluation({ samplesPath, skillDir, variantSpecs = [],
         requires,
         layeredStats,
         repeat,
+        holdoutRatio,
         batch,
         judgeRepeat,
         judgeModels,
@@ -366,7 +367,7 @@ export function buildVarianceData(runs, bootstrapSamples = DEFAULT_BOOTSTRAP_SAM
         ...(saturation ? { saturation } : {}),
     };
 }
-export async function runBatchEvaluation({ skillDir, model = DEFAULT_MODEL, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, noJudge = false, dryRun = false, concurrency = 1, timeoutMs, executorName = 'claude', jobStore = null, persistJob = true, onProgress = null, onSkillProgress = null, skipConnectivity = false, skipDoctor = false, lang = 'zh', mcpConfig, verbose = false, repeat, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, noCache = false, strictBaseline, variantAllowedSkills, }) {
+export async function runBatchEvaluation({ skillDir, model = DEFAULT_MODEL, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, noJudge = false, dryRun = false, concurrency = 1, timeoutMs, executorName = 'claude', jobStore = null, persistJob = true, onProgress = null, onSkillProgress = null, skipConnectivity = false, skipDoctor = false, lang = 'zh', mcpConfig, verbose = false, repeat, holdoutRatio, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, noCache = false, strictBaseline, variantAllowedSkills, }) {
     // Same unified judge derivation as runEvaluation (downstream pipeline / report build
     // still uses single judgeModel + judgeExecutorName per call).
     const effectiveJudgeModels = judgeModels && judgeModels.length > 0
@@ -485,12 +486,14 @@ export async function runBatchEvaluation({ skillDir, model = DEFAULT_MODEL, outp
         strictBaseline,
         variantAllowedSkills,
         runSingleEvaluation: async (options) => {
-            // repeat > 1 时走 runMultiple 做 variance; batch=true 标记让 meta.request 如实反映
+            // repeat > 1 时走 runMultiple 做 variance; batch=true 标记让 meta.request 如实反映。
+            // holdoutRatio 显式转发(executeBatchEvaluationRuns 不一定把它塞进 options),否则
+            // batch 子报告 meta.request.holdoutRatio 丢失、过拟合门控对 batch 静默失效。
             if (repeat && repeat > 1) {
-                const multi = await runMultiple({ ...options, repeat, batch: true, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget });
+                const multi = await runMultiple({ ...options, repeat, holdoutRatio, batch: true, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget });
                 return { report: multi.report, filePath: multi.filePath };
             }
-            const result = await runEvaluation({ ...options, batch: true, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget });
+            const result = await runEvaluation({ ...options, batch: true, holdoutRatio, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget });
             return { report: result.report, filePath: result.filePath };
         },
     });

package/dist/inputs/eval-config.js

@@ -171,6 +171,11 @@ function validateEvalConfig(parsed, configPath) {
     };
     assertPositiveIntOpt('repeat');
     assertPositiveIntOpt('judgeRepeat');
+    if (obj.holdoutRatio !== undefined) {
+        if (typeof obj.holdoutRatio !== 'number' || !Number.isFinite(obj.holdoutRatio) || obj.holdoutRatio <= 0 || obj.holdoutRatio >= 1) {
+            throw new Error(`${configPath}: holdoutRatio must be a number in (0, 1)`);
+        }
+    }
     if (obj.bootstrapSamples !== undefined) {
         if (typeof obj.bootstrapSamples !== 'number' || !Number.isFinite(obj.bootstrapSamples) || obj.bootstrapSamples < 100) {
             throw new Error(`${configPath}: bootstrapSamples must be a number ≥ 100`);
@@ -256,6 +261,7 @@ function validateEvalConfig(parsed, configPath) {
         variants,
         budget,
         repeat: obj.repeat,
+        holdoutRatio: obj.holdoutRatio,
         judgeRepeat: obj.judgeRepeat,
         bootstrap: obj.bootstrap,
         bootstrapSamples: obj.bootstrapSamples,

package/dist/renderer/summary.js

@@ -101,6 +101,30 @@ function computeMedianCVPercent(report) {
     const stab = medianStabilityCV(report);
     return stab ? stab.cv * 100 : null;
 }
+// Verdict caveats（过拟合 / 知识缺口）渲染进 pill —— 让 HTML 报告和 CLI 说同一件事:
+// CLI 在 verbose rationale 里给这两条,HTML 之前只剩一个泛化后的 level、看不到触发原因。
+// 用 result.caveats 的结构化数据 i18n,而不是重解析 zh rationale 串。
+function renderVerdictCaveats(caveats, lang) {
+    if (!caveats)
+        return '';
+    const lines = [];
+    if (caveats.overfitting) {
+        const c = caveats.overfitting;
+        lines.push(lang === 'zh'
+            ? `⚠ 过拟合敞口:${c.variant} 训练 ${c.trainScore.toFixed(2)} / 留出 ${c.holdoutScore.toFixed(2)}(差 ${c.gap.toFixed(2)}),提升可能不泛化`
+            : `⚠ Overfitting: ${c.variant} train ${c.trainScore.toFixed(2)} / holdout ${c.holdoutScore.toFixed(2)} (gap ${c.gap.toFixed(2)}) — gain may not generalize`);
+    }
+    if (caveats.gapSignal) {
+        const g = caveats.gapSignal;
+        const wm = g.testSetHash ? g.testSetHash.slice(0, 8) : (g.testSetPath ?? '');
+        lines.push(lang === 'zh'
+            ? `知识缺口率 ${g.gapRatePct}%(test set ${wm},informational)`
+            : `Knowledge gap ${g.gapRatePct}% (test set ${wm}, informational)`);
+    }
+    if (lines.length === 0)
+        return '';
+    return `<div class="page-verdict-caveats">${lines.map((l) => `<span class="page-verdict-caveat">${e(l)}</span>`).join('')}</div>`;
+}
 export function renderVerdictPill(report, lang) {
     let result;
     try {
@@ -110,8 +134,16 @@ export function renderVerdictPill(report, lang) {
         return '';
     }
     const level = result.level;
-    const pair = result.perPair?.[0];
+    // representative = top-level worst pair（与 CLI 同口径）,不是第一对。多 treatment 报告里
+    // worst pair 不一定是 perPair[0],用它才不会把错的 treatment 名写进结论。fallback 兼容旧路径。
+    const pair = result.representative ?? result.perPair?.[0];
     const oneLine = verdictOneLine(level, lang, pair?.treatment, pair?.control);
+    // Δ/CI 证据必须跟文案指同一对:按 representative 匹配对应的 pairComparison(alpha 也走这对),
+    // 否则多 treatment 报告会出现「文案 t2、数字 t1」的混搭。匹配不到 / 无 representative 时 fallback [0]。
+    const pairComparisons = report.meta?.pairComparisons;
+    const activeComparison = (pair
+        ? pairComparisons?.find((p) => p.treatment === pair.treatment && p.control === pair.control)
+        : undefined) ?? pairComparisons?.[0];
     const tooltip = levelTooltip(level, lang);
     const prefix = lang === 'zh' ? '测评结论' : 'Verdict';
     // 机器可读 enum 永远是 level token; 显示给用户的文字按 lang i18n.
@@ -120,7 +152,7 @@ export function renderVerdictPill(report, lang) {
     // hero 只放「答案」: 分差是 verdict 的核心证据数字, 单独一枚 chip。
     // 评测规模 (用例数 × 轮次) 走「实验配置」section 的 subtitle 那条 canonical 路径,
     // 不在 hero 里重复; CV / CI 走 chip tooltip + 方法学审计 / 波动表。
-    const ci = report.meta?.pairComparisons?.[0]?.diffBootstrapCI;
+    const ci = activeComparison?.diffBootstrapCI;
     const cvPct = computeMedianCVPercent(report);
     const metrics = [];
     if (ci) {
@@ -128,7 +160,7 @@ export function renderVerdictPill(report, lang) {
         const cvSuffix = cvPct != null
             ? (lang === 'zh' ? `;多轮稳定性 CV=${cvPct.toFixed(1)}% (${cvPct < 5 ? '稳' : cvPct <= STABILITY_UNSTABLE_CV * 100 ? '中' : '不稳'})` : `; CV=${cvPct.toFixed(1)}% (${cvPct < 5 ? 'stable' : cvPct <= STABILITY_UNSTABLE_CV * 100 ? 'moderate' : 'unstable'})`)
             : '';
-        const pctLabel = ciLevelLabel(report.meta?.pairComparisons?.[0]?.alpha);
+        const pctLabel = ciLevelLabel(activeComparison?.alpha);
         const ciTipBase = lang === 'zh'
             ? `实验组与对照组综合分均值差(Δ)。bootstrap ${pctLabel} 可信区间 [${ci.low}, ${ci.high}]，${ci.significant ? '不含 0 = 差异显著' : '跨过 0 = 差异不显著'}${cvSuffix}`
             : `Treatment minus control mean composite score (Δ). Bootstrap ${pctLabel} CI [${ci.low}, ${ci.high}], ${ci.significant ? 'excludes 0 ⇒ significant' : 'spans 0 ⇒ not significant'}${cvSuffix}`;
@@ -146,6 +178,7 @@ export function renderVerdictPill(report, lang) {
       <span class="page-verdict-badge"><span class="page-verdict-badge-dot" aria-hidden="true">●</span>${e(levelDisplay)}</span>
       <span class="page-verdict-text">${e(oneLine)}</span>
     </div>
+    ${renderVerdictCaveats(result.caveats, lang)}
     ${metricChips ? `<div class="page-verdict-metrics">${metricChips}</div>` : ''}
   </section>`;
 }

package/dist/types/eval.d.ts

@@ -220,6 +220,9 @@ export interface EvalConfig {
     budget?: EvalBudget;
     /** --repeat N. Multi-run variance analysis. */
     repeat?: number;
+    /** --holdout-ratio R (0 < R < 1). Hold out a deterministic sample slice and
+     *  report train vs holdout composite as a generalization / overfitting signal. */
+    holdoutRatio?: number;
     /** --judge-repeat N. Each (sample × dimension) judged N times for self-consistency stddev. */
     judgeRepeat?: number;
     /** --bootstrap. Distribution-free CI per variant + pairwise diff. */
@@ -257,6 +260,10 @@ export interface EvaluationRequest {
     dryRun: boolean;
     /** --repeat N; 1 表示单次跑,> 1 走 runMultiple 做 variance 分析 */
     repeat?: number;
+    /** --holdout-ratio R; 0 / 缺省表示不切分(默认)。> 0 时 report-finalize 在结果上
+     *  post-hoc 切出 train / holdout 子集算综合分(`report.analysis.holdout`),供 verdict
+     *  的过拟合门控读取。see src/eval-core/holdout.ts */
+    holdoutRatio?: number;
     /** --batch; default absent/false. True means skill-batch mode. */
     batch?: boolean;
     /** --judge-repeat N; 每条 sample × dimension 用 LLM judge 跑 N 次, 输出 stddev. 默认 1 (单次). */

package/dist/types/report.d.ts

@@ -480,6 +480,34 @@ export interface AnalysisResult {
      *  (capability / difficulty / construct / provenance); persisted on report
      *  for studio to surface coverage gaps. See docs/specs/sample-design-spec.md. */
     sampleQuality?: SampleQualityAggregate;
+    /** Opt-in train/holdout generalization breakdown (`omk eval --holdout-ratio`).
+     *  Absent on default runs; present only when a holdout ratio was requested. */
+    holdout?: HoldoutBreakdown;
+}
+/** Train vs holdout composite breakdown for `omk eval --holdout-ratio`.
+ *  Computed post-hoc from `report.results` by `computeHoldoutBreakdown`
+ *  (`src/eval-core/holdout.ts`), sharing the same testSetHash watermark as
+ *  gapReports (gap-spec §7.1). A large train − holdout composite gap is the
+ *  sample-set-overfitting signal the verdict's overfitting gate reads. */
+export interface HoldoutBreakdown {
+    /** Held-out fraction requested via --holdout-ratio. */
+    ratio: number;
+    /** true when either side fell below the minimum subset size → scored full-set,
+     *  no usable split. `perVariant` is empty and the verdict gate stays inert. */
+    disabled?: boolean;
+    /** Per-variant train vs holdout composite (1-5 scale). `*Count` is the authored
+     *  split size; `*Scorable` is how many of those actually produced a composite (> 0)
+     *  — they diverge under partial errors, and the overfitting gate trusts `*Scorable`. */
+    perVariant: Record<string, {
+        trainScore: number;
+        holdoutScore: number;
+        trainCount: number;
+        holdoutCount: number;
+        trainScorable: number;
+        holdoutScorable: number;
+    }>;
+    testSetPath?: string | null;
+    testSetHash?: string | null;
 }
 /** Aggregated sample design coverage stats. Built by
  *  `buildSampleQualityAggregate(samples)` from `Sample.capability` /
@@ -502,6 +530,27 @@ export interface SampleQualityAggregate {
     sampleCountWithDifficulty: number;
     sampleCountWithConstruct: number;
     sampleCountWithProvenance: number;
+    /** Relative-balance / skew of the sample set (derived from the distributions
+     *  above). Flags over-representation — "70% of samples are easy" — without an
+     *  external denominator. Diagnostic only; never feeds grading / judge / verdict. */
+    representativeness?: Representativeness;
+}
+/** Distribution skew over what the sample set declares. Pure relative balance —
+ *  there is no authored "expected" capability list to measure absolute coverage
+ *  against (capabilities are free-form strings), so this reports concentration
+ *  (dominant bucket share, 0-1) and the dominant label per dimension. */
+export interface Representativeness {
+    /** Distinct capabilities declared across the set. */
+    capabilityCount: number;
+    /** Dominant capability's share of all capability tags (0-1); 0 when none declared. */
+    capabilityConcentration: number;
+    dominantCapability?: string;
+    /** Dominant difficulty bucket's share of samples that declared a difficulty (0-1). */
+    difficultyConcentration: number;
+    dominantDifficulty?: 'easy' | 'medium' | 'hard';
+    /** Dominant construct's share of samples that declared a construct (0-1). */
+    constructConcentration: number;
+    dominantConstruct?: string;
 }
 export interface HedgingVerdict {
     isUncertainty: boolean;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "oh-my-knowledge",
-  "version": "0.41.0",
+  "version": "0.42.0",
   "packageManager": "yarn@4.16.0",
   "description": "Evaluation framework for LLM knowledge inputs — prompts, RAG corpora, skills, agent workflows. Fix the model, vary the artifact. Built-in statistical rigor: bootstrap CI, Krippendorff α, length-debias, saturation curves.",
   "type": "module",