oh-my-knowledge 0.41.0 → 0.42.0

package/dist/cli/lib/i18n-dict/run.js

@@ -51,6 +51,10 @@ export const runDict = {
         zh: '⚠ --judge-repeat "{value}" 无效 (期望 ≥ 1 的整数), 已按 1 次 judge 执行\n',
         en: '⚠ --judge-repeat "{value}" is invalid (expected an integer ≥ 1), falling back to 1 judge call\n',
     },
+    'cli.run.invalid_holdout_ratio': {
+        zh: '⚠ --holdout-ratio "{value}" 无效 (期望 0 到 1 之间的小数), 已忽略、不做 holdout 切分\n',
+        en: '⚠ --holdout-ratio "{value}" is invalid (expected a fraction in (0, 1)), ignored — no holdout split\n',
+    },
     'cli.run.no_debias_length_active': {
         zh: 'ℹ --no-debias-length 已生效：judge prompt 去掉长度去偏指令（debias-off 变体），hash 与默认开启时不同。\n',
         en: 'ℹ --no-debias-length is active: the judge prompt drops the length-debias instruction (debias-off variant); its hash differs from the default.\n',

package/dist/cli/lib/parse-run-config.d.ts

@@ -45,6 +45,9 @@ export interface RunConfig {
     retry?: number;
     resume?: string;
     layeredStats?: boolean;
+    /** --holdout-ratio R (0 < R < 1). Hold out a deterministic sample slice; report-finalize
+     *  computes train vs holdout composite (report.analysis.holdout) for the overfitting gate. */
+    holdoutRatio?: number;
     /** --judge-repeat N. Calls LLM judge N times per (sample × dimension). Default 1. */
     judgeRepeat?: number;
     /** Unified judge config. Always non-empty; 1 entry = single judge, ≥ 2 = ensemble.

package/dist/eval-core/evaluation-job.d.ts

@@ -1,5 +1,5 @@
 import type { Artifact, EvaluationErrorCategory, EvaluationJob, EvaluationRequest, EvaluationRun, JudgeConfig } from '../types/index.js';
-export declare function buildEvaluationRequest({ samplesPath, skillDir, artifacts, model, executor, noJudge, concurrency, timeoutMs, noCache, dryRun, project, owner, tags, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }: {
+export declare function buildEvaluationRequest({ samplesPath, skillDir, artifacts, model, executor, noJudge, concurrency, timeoutMs, noCache, dryRun, project, owner, tags, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }: {
     samplesPath: string;
     skillDir: string;
     artifacts: Artifact[];
@@ -14,6 +14,7 @@ export declare function buildEvaluationRequest({ samplesPath, skillDir, artifact
     owner?: string;
     tags?: string[];
     repeat?: number;
+    holdoutRatio?: number;
     batch?: boolean;
     judgeRepeat?: number;
     judgeModels: JudgeConfig[];

package/dist/eval-core/evaluation-job.js

@@ -1,7 +1,7 @@
 function nowIso() {
     return new Date().toISOString();
 }
-export function buildEvaluationRequest({ samplesPath, skillDir, artifacts, model, executor, noJudge, concurrency, timeoutMs, noCache, dryRun, project, owner, tags, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }) {
+export function buildEvaluationRequest({ samplesPath, skillDir, artifacts, model, executor, noJudge, concurrency, timeoutMs, noCache, dryRun, project, owner, tags, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }) {
     return {
         samplesPath,
         skillDir,
@@ -17,6 +17,7 @@ export function buildEvaluationRequest({ samplesPath, skillDir, artifacts, model
         owner,
         tags,
         repeat,
+        holdoutRatio,
         batch,
         judgeRepeat,
         judgeModels,

package/dist/eval-core/evaluation-reporting.js

@@ -61,10 +61,14 @@ export function getCliVersion() {
     return PKG.version;
 }
 export function getGitInfo() {
+    // stdio 静默 stderr:在非 git 目录(如 omk init 出来的 demo)里 rev-parse 会打印
+    // `fatal: not a git repository` 到终端。catch 已把失败兜成 null(报告省略 git 信息),
+    // 这条 fatal 对用户是纯噪声,吞掉它。与 skill-loader 的 GIT_PROBE_STDIO 同口径。
+    const gitProbeStdio = ['ignore', 'pipe', 'ignore'];
     try {
-        const commit = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf-8' }).trim();
-        const branch = execFileSync('git', ['rev-parse', '--abbrev-ref', 'HEAD'], { encoding: 'utf-8' }).trim();
-        const dirty = execFileSync('git', ['status', '--porcelain'], { encoding: 'utf-8' }).trim().length > 0;
+        const commit = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf-8', stdio: gitProbeStdio }).trim();
+        const branch = execFileSync('git', ['rev-parse', '--abbrev-ref', 'HEAD'], { encoding: 'utf-8', stdio: gitProbeStdio }).trim();
+        const dirty = execFileSync('git', ['status', '--porcelain'], { encoding: 'utf-8', stdio: gitProbeStdio }).trim().length > 0;
         return { commit, commitShort: commit.slice(0, 7), branch, dirty };
     }
     catch {

package/dist/eval-core/holdout.d.ts

@@ -0,0 +1,66 @@
+/**
+ * Holdout split + train/holdout composite breakdown.
+ *
+ * Deterministic (no RNG) train/holdout partitioning of a sample set, plus the
+ * subset-composite recompute that lets `omk eval --holdout-ratio` and `omk evolve`
+ * score a variant on a withheld slice using the *same* aggregation as the headline
+ * composite (`buildVariantSummary`). Lives in eval-core so both the eval pipeline
+ * and the authoring/evolve loop depend *down* into it (authoring → eval-core is the
+ * established direction).
+ *
+ * A large train − holdout composite gap is the generalization / sample-set-overfitting
+ * signal the verdict's overfitting gate reads (`src/eval-core/verdict.ts`).
+ */
+import type { Report, HoldoutBreakdown } from '../types/index.js';
+/** A train / holdout partition of a sample set. */
+export interface HoldoutSplit {
+    trainIds: Set<string>;
+    holdoutIds: Set<string>;
+}
+/** Below this many samples on any side, a split is too small to be meaningful —
+ *  callers fall back to full-set scoring and mark the breakdown `disabled`. */
+export declare const MIN_HOLDOUT_SUBSET = 3;
+/** Pick `count` ids at an even stride across `ids` (deterministic, no RNG) so the
+ *  picked subset is representative of the ordering and stable across rounds/runs. */
+export declare function pickByStride(ids: string[], count: number): Set<string>;
+/**
+ * Deterministically split sample ids into train / holdout by `ratio` (fraction
+ * held out). Holdout members are picked at an even stride so the partition is
+ * representative of the ordering, and the split is stable across rounds and runs
+ * (no RNG). Returns null when ratio ≤ 0 or either side would drop below
+ * MIN_HOLDOUT_SUBSET — the caller then scores on the full set.
+ */
+export declare function splitHoldout(sampleIds: string[], ratio: number): HoldoutSplit | null;
+/**
+ * Mean composite over the subset of a report's results whose sample_id is in
+ * `ids`, using the same aggregation as the full-run summary
+ * (`buildVariantSummary`) so train / holdout scores stay comparable to the
+ * headline composite. Returns 0 when the subset has no scorable entries.
+ */
+export declare function subsetCompositeScore(report: Report, variantKey: string, ids: Set<string>): number;
+/**
+ * How many subset results actually produced a usable composite (> 0) for a variant.
+ * `buildVariantSummary` averages only `compositeScore > 0` entries (schema.ts), so
+ * the mean can rest on far fewer samples than the authored split size when runs
+ * flake / partial-error / budget-abort. The overfitting gate must trust THIS count,
+ * not the authored `trainCount` / `holdoutCount`, or a 1-of-3 holdout gets dressed
+ * up as a 3-sample-backed conclusion.
+ */
+export declare function subsetScorableCount(report: Report, variantKey: string, ids: Set<string>): number;
+/**
+ * Train vs holdout composite breakdown per variant for `omk eval --holdout-ratio`.
+ * Post-hoc — never perturbs the headline aggregation or bootstrap CI.
+ *
+ * The split is taken over `sampleIdOrder` — the **stable authored sample order**
+ * (the loaded `samples` file order), NOT `report.results`, whose insertion order
+ * is the concurrent-completion order and drifts run-to-run. Binding the stride pick
+ * to the authored order is what makes the holdout (and the verdict overfitting gate
+ * it feeds) deterministic and reproducible. Subset scores are then read from
+ * `report.results` by id-set membership, which is order-independent.
+ *
+ * When the split is too small on either side (< MIN_HOLDOUT_SUBSET) it returns
+ * `{ disabled: true }` with an empty `perVariant`, so the verdict overfitting gate
+ * stays inert. The testSetHash watermark (gap-spec §7.1) is attached by the caller,
+ * shared with gapReports.
+ */
+export declare function computeHoldoutBreakdown(report: Report, variantNames: string[], ratio: number, sampleIdOrder: string[]): HoldoutBreakdown;

package/dist/eval-core/holdout.js

@@ -0,0 +1,118 @@
+/**
+ * Holdout split + train/holdout composite breakdown.
+ *
+ * Deterministic (no RNG) train/holdout partitioning of a sample set, plus the
+ * subset-composite recompute that lets `omk eval --holdout-ratio` and `omk evolve`
+ * score a variant on a withheld slice using the *same* aggregation as the headline
+ * composite (`buildVariantSummary`). Lives in eval-core so both the eval pipeline
+ * and the authoring/evolve loop depend *down* into it (authoring → eval-core is the
+ * established direction).
+ *
+ * A large train − holdout composite gap is the generalization / sample-set-overfitting
+ * signal the verdict's overfitting gate reads (`src/eval-core/verdict.ts`).
+ */
+import { buildVariantSummary } from './schema.js';
+/** Below this many samples on any side, a split is too small to be meaningful —
+ *  callers fall back to full-set scoring and mark the breakdown `disabled`. */
+export const MIN_HOLDOUT_SUBSET = 3;
+/** Pick `count` ids at an even stride across `ids` (deterministic, no RNG) so the
+ *  picked subset is representative of the ordering and stable across rounds/runs. */
+export function pickByStride(ids, count) {
+    const picked = new Set();
+    if (count <= 0)
+        return picked;
+    const stride = ids.length / count;
+    for (let k = 0; k < count; k++)
+        picked.add(ids[Math.floor(k * stride)]);
+    return picked;
+}
+/**
+ * Deterministically split sample ids into train / holdout by `ratio` (fraction
+ * held out). Holdout members are picked at an even stride so the partition is
+ * representative of the ordering, and the split is stable across rounds and runs
+ * (no RNG). Returns null when ratio ≤ 0 or either side would drop below
+ * MIN_HOLDOUT_SUBSET — the caller then scores on the full set.
+ */
+export function splitHoldout(sampleIds, ratio) {
+    if (!(ratio > 0) || sampleIds.length === 0)
+        return null;
+    const holdoutCount = Math.round(sampleIds.length * ratio);
+    const trainCount = sampleIds.length - holdoutCount;
+    if (holdoutCount < MIN_HOLDOUT_SUBSET || trainCount < MIN_HOLDOUT_SUBSET)
+        return null;
+    const holdoutIds = pickByStride(sampleIds, holdoutCount);
+    const trainIds = new Set(sampleIds.filter((id) => !holdoutIds.has(id)));
+    return { trainIds, holdoutIds };
+}
+/**
+ * Mean composite over the subset of a report's results whose sample_id is in
+ * `ids`, using the same aggregation as the full-run summary
+ * (`buildVariantSummary`) so train / holdout scores stay comparable to the
+ * headline composite. Returns 0 when the subset has no scorable entries.
+ */
+export function subsetCompositeScore(report, variantKey, ids) {
+    const entries = [];
+    for (const r of report.results) {
+        if (!ids.has(r.sample_id))
+            continue;
+        const v = r.variants[variantKey];
+        if (v)
+            entries.push(v);
+    }
+    if (entries.length === 0)
+        return 0;
+    return buildVariantSummary(entries).avgCompositeScore ?? 0;
+}
+/**
+ * How many subset results actually produced a usable composite (> 0) for a variant.
+ * `buildVariantSummary` averages only `compositeScore > 0` entries (schema.ts), so
+ * the mean can rest on far fewer samples than the authored split size when runs
+ * flake / partial-error / budget-abort. The overfitting gate must trust THIS count,
+ * not the authored `trainCount` / `holdoutCount`, or a 1-of-3 holdout gets dressed
+ * up as a 3-sample-backed conclusion.
+ */
+export function subsetScorableCount(report, variantKey, ids) {
+    let n = 0;
+    for (const r of report.results) {
+        if (!ids.has(r.sample_id))
+            continue;
+        const v = r.variants[variantKey];
+        if (v && typeof v.compositeScore === 'number' && v.compositeScore > 0)
+            n++;
+    }
+    return n;
+}
+/**
+ * Train vs holdout composite breakdown per variant for `omk eval --holdout-ratio`.
+ * Post-hoc — never perturbs the headline aggregation or bootstrap CI.
+ *
+ * The split is taken over `sampleIdOrder` — the **stable authored sample order**
+ * (the loaded `samples` file order), NOT `report.results`, whose insertion order
+ * is the concurrent-completion order and drifts run-to-run. Binding the stride pick
+ * to the authored order is what makes the holdout (and the verdict overfitting gate
+ * it feeds) deterministic and reproducible. Subset scores are then read from
+ * `report.results` by id-set membership, which is order-independent.
+ *
+ * When the split is too small on either side (< MIN_HOLDOUT_SUBSET) it returns
+ * `{ disabled: true }` with an empty `perVariant`, so the verdict overfitting gate
+ * stays inert. The testSetHash watermark (gap-spec §7.1) is attached by the caller,
+ * shared with gapReports.
+ */
+export function computeHoldoutBreakdown(report, variantNames, ratio, sampleIdOrder) {
+    const split = splitHoldout(sampleIdOrder, ratio);
+    if (!split) {
+        return { ratio, disabled: true, perVariant: {} };
+    }
+    const perVariant = {};
+    for (const v of variantNames) {
+        perVariant[v] = {
+            trainScore: Number(subsetCompositeScore(report, v, split.trainIds).toFixed(4)),
+            holdoutScore: Number(subsetCompositeScore(report, v, split.holdoutIds).toFixed(4)),
+            trainCount: split.trainIds.size,
+            holdoutCount: split.holdoutIds.size,
+            trainScorable: subsetScorableCount(report, v, split.trainIds),
+            holdoutScorable: subsetScorableCount(report, v, split.holdoutIds),
+        };
+    }
+    return { ratio, perVariant };
+}

package/dist/eval-core/verdict.d.ts

@@ -27,7 +27,7 @@
  * proven optimal. Each rule's source (NIST AI 800-3 / Krippendorff thresholds /
  * empirical) is documented inline so users can audit and override.
  */
-import type { Report } from '../types/index.js';
+import type { Lang, Report } from '../types/index.js';
 /**
  * Below this sample count a non-significant diff is read as UNDERPOWERED
  * (only large effects are detectable) rather than NOISE. Matches the
@@ -63,6 +63,19 @@ export declare const STABILITY_UNSTABLE_CV = 0.15;
  * doc ↔ code parity guarded by `test/scripts/doc-constants-drift.test.ts`.
  */
 export declare const DEFAULT_GATE_THRESHOLD = 3.5;
+/**
+ * Train − holdout composite gap (1-5 scale) above which `omk eval --holdout-ratio`
+ * is read as **sample-set overfitting**: the gain lives on the samples the skill
+ * was shaped around and does not carry to the held-out slice. Like the stability
+ * gate, a would-be PROGRESS is then downgraded to CAUTIOUS — a win that does not
+ * generalize is not shippable.
+ * **Pragmatic default, not from an external standard**: 0.5 is 10% of the 1-5 scale
+ * — small enough to catch a real generalization drop, wide enough to ignore the
+ * sampling noise of a small holdout slice. Only fires when a holdout split is
+ * present (opt-in), so it never moves a default report's verdict.
+ * doc ↔ code parity guarded by `test/scripts/doc-constants-drift.test.ts`.
+ */
+export declare const OVERFITTING_GAP_THRESHOLD = 0.5;
 export type VerdictLevel = 'PROGRESS' | 'CAUTIOUS' | 'REGRESS' | 'NOISE' | 'UNDERPOWERED' | 'SOLO';
 export interface VerdictResult {
     level: VerdictLevel;
@@ -84,8 +97,37 @@ export interface VerdictResult {
          *  让用户感受到 single-run 的盲区,而不是默默不提。 */
         stability?: string;
         judgeAgreement?: string;
+        /** Overfitting (train vs holdout) caveat — only present under `--holdout-ratio`. */
+        overfitting?: string;
+        /** Knowledge-gap caveat — informational, watermarked, never gates (gap-spec §8). */
+        gapSignal?: string;
         shipRecommendation?: string;
     };
+    /** The pair the top-level verdict is about — the worst pair from the roll-up, NOT
+     *  variants[1]. Surfaces let the HTML pill name the right treatment in a
+     *  control-vs-many report instead of re-deriving from the first pair. Undefined for
+     *  SOLO / pairless reports. */
+    representative?: {
+        control: string;
+        treatment: string;
+    };
+    /** Structured caveats (language-neutral) so HTML / other surfaces can i18n them
+     *  instead of re-parsing the zh `rationale` strings. Present only when the caveat
+     *  fires; mirrors `rationale.overfitting` / `rationale.gapSignal`. */
+    caveats?: {
+        overfitting?: {
+            variant: string;
+            trainScore: number;
+            holdoutScore: number;
+            gap: number;
+        };
+        gapSignal?: {
+            variant: string;
+            gapRatePct: number;
+            testSetPath?: string | null;
+            testSetHash?: string | null;
+        };
+    };
     /** Variants present in the report (best-vs-control framing). */
     variants: string[];
 }
@@ -129,4 +171,5 @@ export declare function medianStabilityCV(report: Report): {
  */
 export declare function formatVerdictText(result: VerdictResult, options?: {
     verbose?: boolean;
+    lang?: Lang;
 }): string;

package/dist/eval-core/verdict.js

@@ -30,6 +30,7 @@
 import { evaluateLayerGates } from './layer-gates.js';
 import { ciLevelLabel } from './bootstrap.js';
 import { analyzeJudgeIndependence } from './judge-independence.js';
+import { MIN_HOLDOUT_SUBSET } from './holdout.js';
 /**
  * Below this sample count a non-significant diff is read as UNDERPOWERED
  * (only large effects are detectable) rather than NOISE. Matches the
@@ -65,6 +66,19 @@ export const STABILITY_UNSTABLE_CV = 0.15;
  * doc ↔ code parity guarded by `test/scripts/doc-constants-drift.test.ts`.
  */
 export const DEFAULT_GATE_THRESHOLD = 3.5;
+/**
+ * Train − holdout composite gap (1-5 scale) above which `omk eval --holdout-ratio`
+ * is read as **sample-set overfitting**: the gain lives on the samples the skill
+ * was shaped around and does not carry to the held-out slice. Like the stability
+ * gate, a would-be PROGRESS is then downgraded to CAUTIOUS — a win that does not
+ * generalize is not shippable.
+ * **Pragmatic default, not from an external standard**: 0.5 is 10% of the 1-5 scale
+ * — small enough to catch a real generalization drop, wide enough to ignore the
+ * sampling noise of a small holdout slice. Only fires when a holdout split is
+ * present (opt-in), so it never moves a default report's verdict.
+ * doc ↔ code parity guarded by `test/scripts/doc-constants-drift.test.ts`.
+ */
+export const OVERFITTING_GAP_THRESHOLD = 0.5;
 /**
  * Compute a verdict for a finished report. Pure function — no I/O.
  */
@@ -79,17 +93,28 @@ export function computeVerdict(report, options = {}) {
         const gate = evaluateLayerGates(summary, gateThreshold);
         // SOLO 只有绝对分、无 A/B 差值可抵消自我偏好,故同厂商评委的 caveat 更该出。
         const judgeInd = judgeIndependenceCaveat(report);
+        // 过拟合 / gap 在 SOLO 也有意义(单变体是否泛化 / 缺口多大),但 SOLO 无 PROGRESS 可降,只附提示不门控。
+        const overfit = overfittingCaveat(report);
+        const gap = gapSignalCaveat(report);
         return {
             level: 'SOLO',
             headline: (gate.allPass
                 ? `SOLO · single variant, three-layer gate PASS @ threshold ${gateThreshold}`
-                : `SOLO · single variant, three-layer gate FAIL — see ci output`) + judgeInd.note,
+                : `SOLO · single variant, three-layer gate FAIL — see ci output`) + judgeInd.note + overfit.note + gap.note,
             rationale: {
                 layerWinners: gate.lines.join('; '),
                 sampleSize: `N=${sampleCount}`,
                 stability: formatStability(report),
                 ...(judgeInd.rationale ? { judgeAgreement: judgeInd.rationale } : {}),
+                ...(overfit.rationale ? { overfitting: overfit.rationale } : {}),
+                ...(gap.rationale ? { gapSignal: gap.rationale } : {}),
             },
+            ...((overfit.data || gap.data) ? {
+                caveats: {
+                    ...(overfit.data ? { overfitting: overfit.data } : {}),
+                    ...(gap.data ? { gapSignal: gap.data } : {}),
+                },
+            } : {}),
             variants,
         };
     }
@@ -118,11 +143,17 @@ export function computeVerdict(report, options = {}) {
     // 不再加码,顺序与 worst-case roll-up 一致。
     const stab = medianStabilityCV(report);
     const stabilityGated = topLevel === 'PROGRESS' && stab !== null && stab.cv > STABILITY_UNSTABLE_CV;
-    const level = stabilityGated ? 'CAUTIOUS' : topLevel;
+    // 过拟合门控:与稳定性门控同形——opt-in holdout 下 train/holdout 分差过大 → PROGRESS 降 CAUTIOUS。
+    // overfittingCaveat 首行短路无 holdout 的报告,故默认报告 level 与 headline 逐字节不变。
+    const overfit = overfittingCaveat(report);
+    const overfitGated = topLevel === 'PROGRESS' && overfit.gated;
+    const level = (stabilityGated || overfitGated) ? 'CAUTIOUS' : topLevel;
     const stabilityNote = stabilityGated && stab
         ? ` · 显著但 run-to-run 不稳(CV=${(stab.cv * 100).toFixed(1)}% > ${(STABILITY_UNSTABLE_CV * 100).toFixed(0)}%)`
         : '';
     const judgeInd = judgeIndependenceCaveat(report);
+    // gap 软提示:不改 level(spec §8),低缺口 / 无 gapReports 时空串,headline 逐字节不变。
+    const gap = gapSignalCaveat(report);
     const significance = representative
         ? formatSignificance(representative)
         : 'no pairwise comparison available — was --bootstrap used?';
@@ -134,7 +165,7 @@ export function computeVerdict(report, options = {}) {
     return {
         level,
         headline: representative
-            ? `${level} · ${representative.treatment} vs ${representative.control}: ${representative.headline}${stabilityNote}${judgeInd.note}`
+            ? `${level} · ${representative.treatment} vs ${representative.control}: ${representative.headline}${stabilityNote}${judgeInd.note}${overfit.note}${gap.note}`
             : `${level} · ${variants.length} variants`,
         perPair,
         rationale: {
@@ -143,8 +174,17 @@ export function computeVerdict(report, options = {}) {
             sampleSize,
             stability,
             judgeAgreement,
+            ...(overfit.rationale ? { overfitting: overfit.rationale } : {}),
+            ...(gap.rationale ? { gapSignal: gap.rationale } : {}),
             shipRecommendation,
         },
+        ...(representative ? { representative: { control: representative.control, treatment: representative.treatment } } : {}),
+        ...((overfit.data || gap.data) ? {
+            caveats: {
+                ...(overfit.data ? { overfitting: overfit.data } : {}),
+                ...(gap.data ? { gapSignal: gap.data } : {}),
+            },
+        } : {}),
         variants,
     };
 }
@@ -414,12 +454,125 @@ function judgeIndependenceCaveat(report) {
         rationale: `${reasons.join('；')} —— 绝对分可能偏高；换跨厂商评委(--judge-models)或挂 gold(omk eval gold compare)校准`,
     };
 }
-function recommendation(level, _perPair) {
+/**
+ * Overfitting caveat from the opt-in train/holdout breakdown (`--holdout-ratio`).
+ * `gated` drives a PROGRESS → CAUTIOUS downgrade (a win that does not carry to the
+ * held-out slice is not shippable), mirroring the stability gate. **First line
+ * short-circuits when there is no holdout split**, so default reports (no
+ * `analysis.holdout`) are byte-identical — the gate only ever fires when the user
+ * opted into a holdout, which is brand-new behaviour with no historical reports.
+ */
+/**
+ * The treatment variants a report-level caveat should scan. control = variants[0];
+ * treatments = the rest. For a single-variant (SOLO) report there is no control,
+ * so the lone variant is itself the subject. Scanning **all** treatments (not just
+ * variants[1]) keeps the caveats aligned with the verdict's worst-case roll-up over
+ * `perPair` — a control-vs-many report must not miss the 2nd/3rd treatment.
+ */
+function treatmentVariants(report) {
+    const variants = report.meta?.variants ?? [];
+    return variants.length >= 2 ? variants.slice(1) : variants.slice(0, 1);
+}
+function overfittingCaveat(report) {
+    const holdout = report.analysis?.holdout;
+    if (!holdout || holdout.disabled)
+        return { note: '', gated: false };
+    // Worst-case over all treatments — matches the verdict roll-up. A PROGRESS top-level
+    // means every pair passed, so a single overfitting treatment must still downgrade it.
+    let worst = null;
+    for (const t of treatmentVariants(report)) {
+        const pv = holdout.perVariant[t];
+        if (!pv)
+            continue;
+        // 门控绑「实际可评分条目数」,不是 authored 切分数:某侧 3 条里只有 1 条真出分(其余
+        // error / budget-abort)时,分差只有 1 个样本支撑,把它包装成「3 条结论」会误判过拟合。
+        // 两侧 scorable 都 ≥ MIN_HOLDOUT_SUBSET 才信。这也顺带挡掉 score=0(scorable=0)的测量假象。
+        if (pv.trainScorable < MIN_HOLDOUT_SUBSET || pv.holdoutScorable < MIN_HOLDOUT_SUBSET)
+            continue;
+        const gap = pv.trainScore - pv.holdoutScore;
+        if (gap <= OVERFITTING_GAP_THRESHOLD)
+            continue;
+        if (!worst || gap > worst.gap)
+            worst = { variant: t, train: pv.trainScore, holdout: pv.holdoutScore, gap };
+    }
+    if (!worst)
+        return { note: '', gated: false };
+    return {
+        note: ` · 过拟合敞口(${worst.variant}: train ${worst.train.toFixed(2)} − holdout ${worst.holdout.toFixed(2)} = ${worst.gap.toFixed(2)} > ${OVERFITTING_GAP_THRESHOLD})`,
+        rationale: `${worst.variant} train/holdout 综合分差 ${worst.gap.toFixed(2)} 超阈值 ${OVERFITTING_GAP_THRESHOLD}(holdout ratio ${holdout.ratio}）—— 提升可能是对用例集过拟合、对 holdout 不泛化；扩充用例集或换独立外验集复核`,
+        gated: true,
+        data: { variant: worst.variant, trainScore: worst.train, holdoutScore: worst.holdout, gap: worst.gap },
+    };
+}
+/**
+ * Knowledge-gap rate above which the verdict appends an **informational** caveat.
+ * Internal-only, deliberately NOT exported: gap rate is informational, never a
+ * gate (knowledge-gap-signal-spec.md §8), so exposing this as a tunable constant
+ * would invite reading it as a pass/fail line.
+ */
+const GAP_CAVEAT_BAND = 0.2;
+/**
+ * Knowledge-gap caveat (`report.analysis.gapReports`). **Soft only — never changes
+ * the verdict level** (spec §8: a nudge, not a fail). Surfaces the treatment's gap
+ * rate with its mandatory test-set watermark (spec §7.1: a gap number without a
+ * watermark is invalid output) plus the "informational, not completeness" framing.
+ * Empty note below the band, so the common low-gap case keeps verdict headlines
+ * byte-identical.
+ */
+function gapSignalCaveat(report) {
+    const gapReports = report.analysis?.gapReports;
+    if (!gapReports)
+        return { note: '' };
+    // Worst-case (highest gap rate) over all treatments — a control-vs-many report must
+    // surface the noisiest treatment, not just variants[1]. spec §7.1: gap 数必须带
+    // test-set 水印,否则视为无效输出 —— 无水印的条目不参与(而非吐裸缺口率)。生产报告
+    // 恒带水印(report-finalize 强制),此处只防手搓 / 退化报告。
+    let gr = null;
+    for (const t of treatmentVariants(report)) {
+        const cand = gapReports[t];
+        if (!cand || cand.gapRate < GAP_CAVEAT_BAND)
+            continue;
+        if (!cand.testSetPath && !cand.testSetHash)
+            continue;
+        if (!gr || cand.gapRate > gr.gapRate)
+            gr = cand;
+    }
+    if (!gr)
+        return { note: '' };
+    const pct = (gr.gapRate * 100).toFixed(0);
+    const shortHash = gr.testSetHash ? gr.testSetHash.slice(0, 8) : '';
+    const tag = shortHash || gr.testSetPath;
+    const watermark = gr.testSetPath
+        ? `${gr.testSetPath}${shortHash ? ` @ ${shortHash}` : ''}`
+        : shortHash;
+    return {
+        note: ` · 知识缺口率 ${pct}% @ ${tag}`,
+        rationale: `知识缺口率 ${pct}%(test set: ${watermark}，N=${gr.sampleCount}）—— informational，反映当前用例集与知识库的交互、非完备性度量；高缺口提示扩充知识库或复核未覆盖文件`,
+        data: { variant: gr.variant, gapRatePct: Number(pct), testSetPath: gr.testSetPath, testSetHash: gr.testSetHash },
+    };
+}
+function recommendation(level, _perPair, lang = 'en') {
+    if (lang === 'zh') {
+        switch (level) {
+            case 'PROGRESS':
+                return '可发布 —— 实验组显著更优，且通过所有分层门控。';
+            case 'CAUTIOUS':
+                return '需排查 —— 提升是真的，但至少触发了一条告警（门控破损 / 提升微不足道 / 仅部分恢复 / 评委分歧 / 跨轮不稳 / 训练-留出过拟合）。不要盲发。';
+            case 'REGRESS':
+                return '勿发布 —— 实验组退步。检查最差的那一层，修好再重跑。';
+            case 'NOISE':
+                return '不下结论 —— 差异置信区间跨过 0，当前 N 下分辨不出效果。';
+            case 'UNDERPOWERED':
+                return '数据不足 —— 增加用例数（建议 2× 当前）后重跑。';
+            case 'SOLO':
+                return '缺对照 —— 单变体报告。用 --control baseline --treatment <名字> 重跑。';
+        }
+    }
     switch (level) {
         case 'PROGRESS':
             return 'SHIP — treatment is significantly better and passes all layer gates.';
         case 'CAUTIOUS':
-            return 'INVESTIGATE — the gain is real but at least one warning fired (broken gate, trivially small, partial recovery, judge dissent, or run-to-run unstable). Do not ship blind.';
+            return 'INVESTIGATE — the gain is real but at least one warning fired (broken gate, trivially small, partial recovery, judge dissent, run-to-run unstable, or train/holdout overfitting). Do not ship blind.';
         case 'REGRESS':
             return 'DO NOT SHIP — treatment regresses. Check the worst layer and re-run with the fix.';
         case 'NOISE':
@@ -435,22 +588,31 @@ function recommendation(level, _perPair) {
  *  spec — one verdict + four rationale bullets + one ship recommendation.
  */
 export function formatVerdictText(result, options = {}) {
+    // lang 默认 'en':保留既有英文输出逐字节不变(verdict.test 与历史 CLI 行为)。zh 只本地化
+    //  标签与 ship 建议;headline 是 Δ/CI/N 统计记号 —— 跨语言中性、且会随 report 持久化,
+    //  不翻译(翻它=改可比性锚点)。recommendation 在 format 时按 lang 重新派生,不动 computeVerdict。
+    const zh = options.lang === 'zh';
     const lines = [];
-    lines.push(`Verdict: ${result.level}`);
+    lines.push(zh ? `判定：${result.level}` : `Verdict: ${result.level}`);
     lines.push(`  ${result.headline}`);
     if (result.rationale.layerWinners)
-        lines.push(`  Layer winners: ${result.rationale.layerWinners}`);
+        lines.push(zh ? `  分层优胜：${result.rationale.layerWinners}` : `  Layer winners: ${result.rationale.layerWinners}`);
     if (result.rationale.sampleSize)
-        lines.push(`  Sample size:   ${result.rationale.sampleSize}`);
+        lines.push(zh ? `  用例规模：${result.rationale.sampleSize}` : `  Sample size:   ${result.rationale.sampleSize}`);
     if (result.rationale.stability)
-        lines.push(`  Stability:     ${result.rationale.stability}`);
+        lines.push(zh ? `  跨轮稳定：${result.rationale.stability}` : `  Stability:     ${result.rationale.stability}`);
     if (result.rationale.judgeAgreement)
-        lines.push(`  Judge α:       ${result.rationale.judgeAgreement}`);
-    if (result.rationale.shipRecommendation)
-        lines.push(`  ${result.rationale.shipRecommendation}`);
+        lines.push(zh ? `  评委 α：${result.rationale.judgeAgreement}` : `  Judge α:       ${result.rationale.judgeAgreement}`);
+    if (result.rationale.overfitting)
+        lines.push(zh ? `  过拟合：${result.rationale.overfitting}` : `  Overfitting:   ${result.rationale.overfitting}`);
+    if (result.rationale.gapSignal)
+        lines.push(zh ? `  知识缺口：${result.rationale.gapSignal}` : `  Gap signal:    ${result.rationale.gapSignal}`);
+    if (result.rationale.shipRecommendation) {
+        lines.push(`  ${zh ? recommendation(result.level, [], 'zh') : result.rationale.shipRecommendation}`);
+    }
     if (options.verbose && result.perPair && result.perPair.length > 1) {
         lines.push('');
-        lines.push('  Per-pair detail:');
+        lines.push(zh ? '  逐对明细：' : '  Per-pair detail:');
         for (const p of result.perPair) {
             lines.push(`    ${p.level}: ${p.treatment} vs ${p.control} — ${p.headline}`);
         }

package/dist/eval-workflows/evaluation-pipeline/report-finalize.js

@@ -15,6 +15,7 @@
 import { analyzeResults } from '../../analysis/report-diagnostics.js';
 import { computeReportCoverage } from '../../analysis/coverage-analyzer.js';
 import { computeReportGapRates } from '../../analysis/gap-analyzer.js';
+import { computeHoldoutBreakdown } from '../../eval-core/holdout.js';
 import { computeTestSetHash } from './test-set-hash.js';
 export function finalizeEvaluationReport({ report, results, artifacts, variantNames, samplesPath, samplesSourceFiles, samples, }) {
     // pass samples so analyzeResults can populate analysis.sampleQuality
@@ -33,9 +34,13 @@ export function finalizeEvaluationReport({ report, results, artifacts, variantNa
     // Gap rate computation runs on every successful report regardless of whether
     // tool trace data is present — text-based signals (markers, hedging) still
     // apply. The samples-file SHA is the mandatory watermark required by spec §7.1.
+    // The same hash watermarks the opt-in holdout breakdown below, so it is computed
+    // once and shared (both coverage-class numbers must carry the same test-set id).
+    const holdoutRatio = report.meta?.request?.holdoutRatio ?? 0;
     const gapReports = computeReportGapRates(report.results, variantNames);
+    const needsWatermark = Object.keys(gapReports).length > 0 || holdoutRatio > 0;
+    const testSetHash = needsWatermark ? computeTestSetHash(samplesPath, samplesSourceFiles) : null;
     if (Object.keys(gapReports).length > 0) {
-        const testSetHash = computeTestSetHash(samplesPath, samplesSourceFiles);
         for (const variant of variantNames) {
             const gr = gapReports[variant];
             if (!gr)
@@ -45,5 +50,18 @@ export function finalizeEvaluationReport({ report, results, artifacts, variantNa
         }
         report.analysis.gapReports = gapReports;
     }
+    // Opt-in train/holdout generalization breakdown (`omk eval --holdout-ratio`).
+    // Post-hoc over report.results — never perturbs the headline composite or the
+    // bootstrap CI. A large train − holdout gap is the overfitting signal the verdict
+    // overfitting gate reads (src/eval-core/verdict.ts); absent on default runs.
+    if (holdoutRatio > 0) {
+        // 切分按 samples 的稳定原始顺序(文件顺序),不依赖 report.results 的并发完成落盘顺序,
+        // 否则同批样本在不同并发/时序下 holdout 子集会漂、verdict 过拟合门控跟着漂。
+        const sampleIdOrder = samples.map((s) => s.sample_id);
+        const holdout = computeHoldoutBreakdown(report, variantNames, holdoutRatio, sampleIdOrder);
+        holdout.testSetPath = samplesPath;
+        holdout.testSetHash = testSetHash;
+        report.analysis.holdout = holdout;
+    }
     return report;
 }