oh-my-knowledge 0.40.0 → 0.42.0

package/dist/eval-core/holdout.js

@@ -0,0 +1,118 @@
+/**
+ * Holdout split + train/holdout composite breakdown.
+ *
+ * Deterministic (no RNG) train/holdout partitioning of a sample set, plus the
+ * subset-composite recompute that lets `omk eval --holdout-ratio` and `omk evolve`
+ * score a variant on a withheld slice using the *same* aggregation as the headline
+ * composite (`buildVariantSummary`). Lives in eval-core so both the eval pipeline
+ * and the authoring/evolve loop depend *down* into it (authoring → eval-core is the
+ * established direction).
+ *
+ * A large train − holdout composite gap is the generalization / sample-set-overfitting
+ * signal the verdict's overfitting gate reads (`src/eval-core/verdict.ts`).
+ */
+import { buildVariantSummary } from './schema.js';
+/** Below this many samples on any side, a split is too small to be meaningful —
+ *  callers fall back to full-set scoring and mark the breakdown `disabled`. */
+export const MIN_HOLDOUT_SUBSET = 3;
+/** Pick `count` ids at an even stride across `ids` (deterministic, no RNG) so the
+ *  picked subset is representative of the ordering and stable across rounds/runs. */
+export function pickByStride(ids, count) {
+    const picked = new Set();
+    if (count <= 0)
+        return picked;
+    const stride = ids.length / count;
+    for (let k = 0; k < count; k++)
+        picked.add(ids[Math.floor(k * stride)]);
+    return picked;
+}
+/**
+ * Deterministically split sample ids into train / holdout by `ratio` (fraction
+ * held out). Holdout members are picked at an even stride so the partition is
+ * representative of the ordering, and the split is stable across rounds and runs
+ * (no RNG). Returns null when ratio ≤ 0 or either side would drop below
+ * MIN_HOLDOUT_SUBSET — the caller then scores on the full set.
+ */
+export function splitHoldout(sampleIds, ratio) {
+    if (!(ratio > 0) || sampleIds.length === 0)
+        return null;
+    const holdoutCount = Math.round(sampleIds.length * ratio);
+    const trainCount = sampleIds.length - holdoutCount;
+    if (holdoutCount < MIN_HOLDOUT_SUBSET || trainCount < MIN_HOLDOUT_SUBSET)
+        return null;
+    const holdoutIds = pickByStride(sampleIds, holdoutCount);
+    const trainIds = new Set(sampleIds.filter((id) => !holdoutIds.has(id)));
+    return { trainIds, holdoutIds };
+}
+/**
+ * Mean composite over the subset of a report's results whose sample_id is in
+ * `ids`, using the same aggregation as the full-run summary
+ * (`buildVariantSummary`) so train / holdout scores stay comparable to the
+ * headline composite. Returns 0 when the subset has no scorable entries.
+ */
+export function subsetCompositeScore(report, variantKey, ids) {
+    const entries = [];
+    for (const r of report.results) {
+        if (!ids.has(r.sample_id))
+            continue;
+        const v = r.variants[variantKey];
+        if (v)
+            entries.push(v);
+    }
+    if (entries.length === 0)
+        return 0;
+    return buildVariantSummary(entries).avgCompositeScore ?? 0;
+}
+/**
+ * How many subset results actually produced a usable composite (> 0) for a variant.
+ * `buildVariantSummary` averages only `compositeScore > 0` entries (schema.ts), so
+ * the mean can rest on far fewer samples than the authored split size when runs
+ * flake / partial-error / budget-abort. The overfitting gate must trust THIS count,
+ * not the authored `trainCount` / `holdoutCount`, or a 1-of-3 holdout gets dressed
+ * up as a 3-sample-backed conclusion.
+ */
+export function subsetScorableCount(report, variantKey, ids) {
+    let n = 0;
+    for (const r of report.results) {
+        if (!ids.has(r.sample_id))
+            continue;
+        const v = r.variants[variantKey];
+        if (v && typeof v.compositeScore === 'number' && v.compositeScore > 0)
+            n++;
+    }
+    return n;
+}
+/**
+ * Train vs holdout composite breakdown per variant for `omk eval --holdout-ratio`.
+ * Post-hoc — never perturbs the headline aggregation or bootstrap CI.
+ *
+ * The split is taken over `sampleIdOrder` — the **stable authored sample order**
+ * (the loaded `samples` file order), NOT `report.results`, whose insertion order
+ * is the concurrent-completion order and drifts run-to-run. Binding the stride pick
+ * to the authored order is what makes the holdout (and the verdict overfitting gate
+ * it feeds) deterministic and reproducible. Subset scores are then read from
+ * `report.results` by id-set membership, which is order-independent.
+ *
+ * When the split is too small on either side (< MIN_HOLDOUT_SUBSET) it returns
+ * `{ disabled: true }` with an empty `perVariant`, so the verdict overfitting gate
+ * stays inert. The testSetHash watermark (gap-spec §7.1) is attached by the caller,
+ * shared with gapReports.
+ */
+export function computeHoldoutBreakdown(report, variantNames, ratio, sampleIdOrder) {
+    const split = splitHoldout(sampleIdOrder, ratio);
+    if (!split) {
+        return { ratio, disabled: true, perVariant: {} };
+    }
+    const perVariant = {};
+    for (const v of variantNames) {
+        perVariant[v] = {
+            trainScore: Number(subsetCompositeScore(report, v, split.trainIds).toFixed(4)),
+            holdoutScore: Number(subsetCompositeScore(report, v, split.holdoutIds).toFixed(4)),
+            trainCount: split.trainIds.size,
+            holdoutCount: split.holdoutIds.size,
+            trainScorable: subsetScorableCount(report, v, split.trainIds),
+            holdoutScorable: subsetScorableCount(report, v, split.holdoutIds),
+        };
+    }
+    return { ratio, perVariant };
+}

package/dist/eval-core/judge-independence.d.ts

@@ -0,0 +1,28 @@
+import type { Report } from '../types/index.js';
+import { type ExecutorVendor } from '../executors/shared.js';
+/**
+ * 评委独立性分析(单一来源,verdict caveat 与 analysis 诊断共用)。
+ *
+ * LLM 评委有自我偏好偏置:偏爱与自己同模型家族产出的输出。omk 默认评委(claude:haiku)与默认
+ * 执行器(claude:*)同属一家,敞口默认就开着。本 helper 只算客观事实(评委/输出各属哪家、有没有
+ * 跨厂商评委、是不是单厂商 ensemble、有没有挂 gold 校准),严重度与文案交给消费方(report-diagnostics
+ * / verdict)决定。
+ *
+ * 缓解阶梯(行业共识):换跨厂商评委 > 跨厂商陪审团 > 人工金标校准 > 警告。omk 无法强制换 key,
+ * 故只检测 + 警告并指向 `--judge-models <跨厂商>` 和 `omk eval gold compare`。
+ */
+export interface JudgeIndependence {
+    /** 各评委的厂商家族(与 judgeModels 同序)。 */
+    judgeVendors: ExecutorVendor[];
+    /** 被测输出涉及的厂商家族(去重)。 */
+    outputVendors: ExecutorVendor[];
+    /** 至少有一个评委的厂商不在被测输出厂商集合里 —— 存在独立(跨厂商)评委。 */
+    crossVendorJudgePresent: boolean;
+    /** 有评委、厂商全可归类、且无任何跨厂商评委 —— 自我偏好敞口(J1)。 */
+    sameVendorJudge: boolean;
+    /** ≥ 2 评委且全部同一个(已归类)厂商 —— ensemble 一致性不反驳共有偏置(J2)。 */
+    singleVendorEnsemble: boolean;
+    /** 本次 run 挂了人工 gold 校准(report.meta.humanAgreement 存在)→ 敞口有背板。 */
+    goldCalibrated: boolean;
+}
+export declare function analyzeJudgeIndependence(report: Report): JudgeIndependence;

package/dist/eval-core/judge-independence.js

@@ -0,0 +1,29 @@
+import { executorVendor } from '../executors/shared.js';
+export function analyzeJudgeIndependence(report) {
+    const judges = report.meta?.judgeModels ?? [];
+    const judgeVendors = judges.map((j) => executorVendor(j.executor));
+    const runtimes = report.meta?.executorRuntimes;
+    const outputExecutors = runtimes && Object.keys(runtimes).length > 0
+        ? Object.values(runtimes).map((r) => r.executor).filter((e) => !!e)
+        : (report.meta?.executor ? [report.meta.executor] : []);
+    const outputVendors = [...new Set(outputExecutors.map(executorVendor))];
+    const goldCalibrated = report.meta?.humanAgreement != null;
+    const base = { judgeVendors, outputVendors, goldCalibrated };
+    // 不评估(不误报)的情形:
+    //   - noJudge:judgeModels 为审计保留(非空),但评委根本没跑、composite 无评委层 → 自我偏好 moot;
+    //   - 无评委 / 任一厂商无法归类(自定义 script)/ 拿不到被测厂商。
+    const noJudge = report.meta?.noJudge === true;
+    const anyUnknown = judgeVendors.includes('unknown') || outputVendors.includes('unknown');
+    if (judges.length === 0 || noJudge || anyUnknown || outputVendors.length === 0) {
+        return { ...base, crossVendorJudgePresent: false, sameVendorJudge: false, singleVendorEnsemble: false };
+    }
+    const outSet = new Set(outputVendors);
+    const crossVendorJudgePresent = judgeVendors.some((v) => !outSet.has(v));
+    const singleVendorEnsemble = judges.length >= 2 && new Set(judgeVendors).size === 1;
+    return {
+        ...base,
+        crossVendorJudgePresent,
+        sameVendorJudge: !crossVendorJudgePresent,
+        singleVendorEnsemble,
+    };
+}

package/dist/eval-core/verdict.d.ts

@@ -27,7 +27,7 @@
  * proven optimal. Each rule's source (NIST AI 800-3 / Krippendorff thresholds /
  * empirical) is documented inline so users can audit and override.
  */
-import type { Report } from '../types/index.js';
+import type { Lang, Report } from '../types/index.js';
 /**
  * Below this sample count a non-significant diff is read as UNDERPOWERED
  * (only large effects are detectable) rather than NOISE. Matches the
@@ -55,6 +55,27 @@ export declare const ENSEMBLE_DISSENT_PEARSON = 0.4;
  * never gated by it — see `computeVerdict`.
  */
 export declare const STABILITY_UNSTABLE_CV = 0.15;
+/**
+ * Per-layer pass/fail line for the three-layer gate, on the 1-5 scale.
+ * **Pragmatic default, not derived from an external standard**: 3.5 is a clear margin
+ * above the 3.0 scale midpoint ("basically acceptable"), so a layer must land
+ * comfortably in the upper half to pass. Overridable via `omk eval --threshold`.
+ * doc ↔ code parity guarded by `test/scripts/doc-constants-drift.test.ts`.
+ */
+export declare const DEFAULT_GATE_THRESHOLD = 3.5;
+/**
+ * Train − holdout composite gap (1-5 scale) above which `omk eval --holdout-ratio`
+ * is read as **sample-set overfitting**: the gain lives on the samples the skill
+ * was shaped around and does not carry to the held-out slice. Like the stability
+ * gate, a would-be PROGRESS is then downgraded to CAUTIOUS — a win that does not
+ * generalize is not shippable.
+ * **Pragmatic default, not from an external standard**: 0.5 is 10% of the 1-5 scale
+ * — small enough to catch a real generalization drop, wide enough to ignore the
+ * sampling noise of a small holdout slice. Only fires when a holdout split is
+ * present (opt-in), so it never moves a default report's verdict.
+ * doc ↔ code parity guarded by `test/scripts/doc-constants-drift.test.ts`.
+ */
+export declare const OVERFITTING_GAP_THRESHOLD = 0.5;
 export type VerdictLevel = 'PROGRESS' | 'CAUTIOUS' | 'REGRESS' | 'NOISE' | 'UNDERPOWERED' | 'SOLO';
 export interface VerdictResult {
     level: VerdictLevel;
@@ -76,13 +97,42 @@ export interface VerdictResult {
          *  让用户感受到 single-run 的盲区,而不是默默不提。 */
         stability?: string;
         judgeAgreement?: string;
+        /** Overfitting (train vs holdout) caveat — only present under `--holdout-ratio`. */
+        overfitting?: string;
+        /** Knowledge-gap caveat — informational, watermarked, never gates (gap-spec §8). */
+        gapSignal?: string;
         shipRecommendation?: string;
     };
+    /** The pair the top-level verdict is about — the worst pair from the roll-up, NOT
+     *  variants[1]. Surfaces let the HTML pill name the right treatment in a
+     *  control-vs-many report instead of re-deriving from the first pair. Undefined for
+     *  SOLO / pairless reports. */
+    representative?: {
+        control: string;
+        treatment: string;
+    };
+    /** Structured caveats (language-neutral) so HTML / other surfaces can i18n them
+     *  instead of re-parsing the zh `rationale` strings. Present only when the caveat
+     *  fires; mirrors `rationale.overfitting` / `rationale.gapSignal`. */
+    caveats?: {
+        overfitting?: {
+            variant: string;
+            trainScore: number;
+            holdoutScore: number;
+            gap: number;
+        };
+        gapSignal?: {
+            variant: string;
+            gapRatePct: number;
+            testSetPath?: string | null;
+            testSetHash?: string | null;
+        };
+    };
     /** Variants present in the report (best-vs-control framing). */
     variants: string[];
 }
 export interface VerdictOptions {
-    /** Three-layer ci-gate threshold; defaults to 3.5 (matches `omk eval`). */
+    /** Three-layer ci-gate threshold; defaults to DEFAULT_GATE_THRESHOLD (matches `omk eval`). */
     gateThreshold?: number;
     /**
      * Magnitude (in raw score points) below which a "significant" diff is treated
@@ -121,4 +171,5 @@ export declare function medianStabilityCV(report: Report): {
  */
 export declare function formatVerdictText(result: VerdictResult, options?: {
     verbose?: boolean;
+    lang?: Lang;
 }): string;

package/dist/eval-core/verdict.js

@@ -29,6 +29,8 @@
  */
 import { evaluateLayerGates } from './layer-gates.js';
 import { ciLevelLabel } from './bootstrap.js';
+import { analyzeJudgeIndependence } from './judge-independence.js';
+import { MIN_HOLDOUT_SUBSET } from './holdout.js';
 /**
  * Below this sample count a non-significant diff is read as UNDERPOWERED
  * (only large effects are detectable) rather than NOISE. Matches the
@@ -56,11 +58,32 @@ export const ENSEMBLE_DISSENT_PEARSON = 0.4;
  * never gated by it — see `computeVerdict`.
  */
 export const STABILITY_UNSTABLE_CV = 0.15;
+/**
+ * Per-layer pass/fail line for the three-layer gate, on the 1-5 scale.
+ * **Pragmatic default, not derived from an external standard**: 3.5 is a clear margin
+ * above the 3.0 scale midpoint ("basically acceptable"), so a layer must land
+ * comfortably in the upper half to pass. Overridable via `omk eval --threshold`.
+ * doc ↔ code parity guarded by `test/scripts/doc-constants-drift.test.ts`.
+ */
+export const DEFAULT_GATE_THRESHOLD = 3.5;
+/**
+ * Train − holdout composite gap (1-5 scale) above which `omk eval --holdout-ratio`
+ * is read as **sample-set overfitting**: the gain lives on the samples the skill
+ * was shaped around and does not carry to the held-out slice. Like the stability
+ * gate, a would-be PROGRESS is then downgraded to CAUTIOUS — a win that does not
+ * generalize is not shippable.
+ * **Pragmatic default, not from an external standard**: 0.5 is 10% of the 1-5 scale
+ * — small enough to catch a real generalization drop, wide enough to ignore the
+ * sampling noise of a small holdout slice. Only fires when a holdout split is
+ * present (opt-in), so it never moves a default report's verdict.
+ * doc ↔ code parity guarded by `test/scripts/doc-constants-drift.test.ts`.
+ */
+export const OVERFITTING_GAP_THRESHOLD = 0.5;
 /**
  * Compute a verdict for a finished report. Pure function — no I/O.
  */
 export function computeVerdict(report, options = {}) {
-    const { gateThreshold = 3.5, triviallySmallDiff = 0.1 } = options;
+    const { gateThreshold = DEFAULT_GATE_THRESHOLD, triviallySmallDiff = 0.1 } = options;
     const variants = report.meta?.variants ?? [];
     const summary = report.summary ?? {};
     const sampleCount = report.meta?.sampleCount ?? 0;
@@ -68,16 +91,30 @@ export function computeVerdict(report, options = {}) {
         // Single-variant — no comparison possible. Just report whether the variant
         // passes its own three-layer gate.
         const gate = evaluateLayerGates(summary, gateThreshold);
+        // SOLO 只有绝对分、无 A/B 差值可抵消自我偏好,故同厂商评委的 caveat 更该出。
+        const judgeInd = judgeIndependenceCaveat(report);
+        // 过拟合 / gap 在 SOLO 也有意义(单变体是否泛化 / 缺口多大),但 SOLO 无 PROGRESS 可降,只附提示不门控。
+        const overfit = overfittingCaveat(report);
+        const gap = gapSignalCaveat(report);
         return {
             level: 'SOLO',
-            headline: gate.allPass
+            headline: (gate.allPass
                 ? `SOLO · single variant, three-layer gate PASS @ threshold ${gateThreshold}`
-                : `SOLO · single variant, three-layer gate FAIL — see ci output`,
+                : `SOLO · single variant, three-layer gate FAIL — see ci output`) + judgeInd.note + overfit.note + gap.note,
             rationale: {
                 layerWinners: gate.lines.join('; '),
                 sampleSize: `N=${sampleCount}`,
                 stability: formatStability(report),
+                ...(judgeInd.rationale ? { judgeAgreement: judgeInd.rationale } : {}),
+                ...(overfit.rationale ? { overfitting: overfit.rationale } : {}),
+                ...(gap.rationale ? { gapSignal: gap.rationale } : {}),
             },
+            ...((overfit.data || gap.data) ? {
+                caveats: {
+                    ...(overfit.data ? { overfitting: overfit.data } : {}),
+                    ...(gap.data ? { gapSignal: gap.data } : {}),
+                },
+            } : {}),
             variants,
         };
     }
@@ -106,22 +143,29 @@ export function computeVerdict(report, options = {}) {
     // 不再加码,顺序与 worst-case roll-up 一致。
     const stab = medianStabilityCV(report);
     const stabilityGated = topLevel === 'PROGRESS' && stab !== null && stab.cv > STABILITY_UNSTABLE_CV;
-    const level = stabilityGated ? 'CAUTIOUS' : topLevel;
+    // 过拟合门控:与稳定性门控同形——opt-in holdout 下 train/holdout 分差过大 → PROGRESS 降 CAUTIOUS。
+    // overfittingCaveat 首行短路无 holdout 的报告,故默认报告 level 与 headline 逐字节不变。
+    const overfit = overfittingCaveat(report);
+    const overfitGated = topLevel === 'PROGRESS' && overfit.gated;
+    const level = (stabilityGated || overfitGated) ? 'CAUTIOUS' : topLevel;
     const stabilityNote = stabilityGated && stab
         ? ` · 显著但 run-to-run 不稳(CV=${(stab.cv * 100).toFixed(1)}% > ${(STABILITY_UNSTABLE_CV * 100).toFixed(0)}%)`
         : '';
+    const judgeInd = judgeIndependenceCaveat(report);
+    // gap 软提示:不改 level(spec §8),低缺口 / 无 gapReports 时空串,headline 逐字节不变。
+    const gap = gapSignalCaveat(report);
     const significance = representative
         ? formatSignificance(representative)
         : 'no pairwise comparison available — was --bootstrap used?';
     const layerWinners = formatLayerWinners(summary, variants);
     const sampleSize = formatSampleSize(report);
     const stability = formatStability(report);
-    const judgeAgreement = formatJudgeAgreement(report);
+    const judgeAgreement = [formatJudgeAgreement(report), judgeInd.rationale].filter(Boolean).join(' · ') || undefined;
     const shipRecommendation = recommendation(level, perPair);
     return {
         level,
         headline: representative
-            ? `${level} · ${representative.treatment} vs ${representative.control}: ${representative.headline}${stabilityNote}`
+            ? `${level} · ${representative.treatment} vs ${representative.control}: ${representative.headline}${stabilityNote}${judgeInd.note}${overfit.note}${gap.note}`
             : `${level} · ${variants.length} variants`,
         perPair,
         rationale: {
@@ -130,8 +174,17 @@ export function computeVerdict(report, options = {}) {
             sampleSize,
             stability,
             judgeAgreement,
+            ...(overfit.rationale ? { overfitting: overfit.rationale } : {}),
+            ...(gap.rationale ? { gapSignal: gap.rationale } : {}),
             shipRecommendation,
         },
+        ...(representative ? { representative: { control: representative.control, treatment: representative.treatment } } : {}),
+        ...((overfit.data || gap.data) ? {
+            caveats: {
+                ...(overfit.data ? { overfitting: overfit.data } : {}),
+                ...(gap.data ? { gapSignal: gap.data } : {}),
+            },
+        } : {}),
         variants,
     };
 }
@@ -376,12 +429,150 @@ function formatJudgeAgreement(report) {
                     : 'poor';
     return `α=${Number.isNaN(a.alpha) ? 'NaN' : a.alpha.toFixed(2)} (${verdict}) vs gold ${a.goldAnnotator}`;
 }
-function recommendation(level, _perPair) {
+/**
+ * 评委独立性 caveat(自我偏好 J1 / 单厂商 ensemble J2)。返回 { note, rationale }:
+ *   note      —— 追到 headline 的短提示(仅未 gold 校准时出,提醒读者敞口);
+ *   rationale —— 并进 rationale.judgeAgreement 的可执行说明(指向跨厂商评委 / gold)。
+ * **不改 verdict level**:omk 固定模型,自我偏好对 baseline / treatment 两臂同等加成、在 verdict
+ * 在意的 A/B 差值里大幅抵消,不该翻 ship/no-ship;真正受影响的是绝对分 / 版本曲线 / 跨模型比较。
+ */
+function judgeIndependenceCaveat(report) {
+    const ind = analyzeJudgeIndependence(report);
+    const reasons = [];
+    if (ind.sameVendorJudge)
+        reasons.push(`评委与被测输出同厂商(${ind.outputVendors.join('/')})`);
+    if (ind.singleVendorEnsemble)
+        reasons.push(`${ind.judgeVendors.length} 个评委同厂商，ensemble 一致性不反驳同模型偏置`);
+    if (reasons.length === 0)
+        return { note: '' };
+    if (ind.goldCalibrated) {
+        // 有 gold 校准背板 → 不进 headline,只软提示。
+        return { note: '', rationale: `自我偏好敞口（${reasons.join('；')}）已有 gold 校准背板` };
+    }
+    return {
+        note: ' · 评委自我偏好敞口未校准',
+        rationale: `${reasons.join('；')} —— 绝对分可能偏高；换跨厂商评委(--judge-models)或挂 gold(omk eval gold compare)校准`,
+    };
+}
+/**
+ * Overfitting caveat from the opt-in train/holdout breakdown (`--holdout-ratio`).
+ * `gated` drives a PROGRESS → CAUTIOUS downgrade (a win that does not carry to the
+ * held-out slice is not shippable), mirroring the stability gate. **First line
+ * short-circuits when there is no holdout split**, so default reports (no
+ * `analysis.holdout`) are byte-identical — the gate only ever fires when the user
+ * opted into a holdout, which is brand-new behaviour with no historical reports.
+ */
+/**
+ * The treatment variants a report-level caveat should scan. control = variants[0];
+ * treatments = the rest. For a single-variant (SOLO) report there is no control,
+ * so the lone variant is itself the subject. Scanning **all** treatments (not just
+ * variants[1]) keeps the caveats aligned with the verdict's worst-case roll-up over
+ * `perPair` — a control-vs-many report must not miss the 2nd/3rd treatment.
+ */
+function treatmentVariants(report) {
+    const variants = report.meta?.variants ?? [];
+    return variants.length >= 2 ? variants.slice(1) : variants.slice(0, 1);
+}
+function overfittingCaveat(report) {
+    const holdout = report.analysis?.holdout;
+    if (!holdout || holdout.disabled)
+        return { note: '', gated: false };
+    // Worst-case over all treatments — matches the verdict roll-up. A PROGRESS top-level
+    // means every pair passed, so a single overfitting treatment must still downgrade it.
+    let worst = null;
+    for (const t of treatmentVariants(report)) {
+        const pv = holdout.perVariant[t];
+        if (!pv)
+            continue;
+        // 门控绑「实际可评分条目数」,不是 authored 切分数:某侧 3 条里只有 1 条真出分(其余
+        // error / budget-abort)时,分差只有 1 个样本支撑,把它包装成「3 条结论」会误判过拟合。
+        // 两侧 scorable 都 ≥ MIN_HOLDOUT_SUBSET 才信。这也顺带挡掉 score=0(scorable=0)的测量假象。
+        if (pv.trainScorable < MIN_HOLDOUT_SUBSET || pv.holdoutScorable < MIN_HOLDOUT_SUBSET)
+            continue;
+        const gap = pv.trainScore - pv.holdoutScore;
+        if (gap <= OVERFITTING_GAP_THRESHOLD)
+            continue;
+        if (!worst || gap > worst.gap)
+            worst = { variant: t, train: pv.trainScore, holdout: pv.holdoutScore, gap };
+    }
+    if (!worst)
+        return { note: '', gated: false };
+    return {
+        note: ` · 过拟合敞口(${worst.variant}: train ${worst.train.toFixed(2)} − holdout ${worst.holdout.toFixed(2)} = ${worst.gap.toFixed(2)} > ${OVERFITTING_GAP_THRESHOLD})`,
+        rationale: `${worst.variant} train/holdout 综合分差 ${worst.gap.toFixed(2)} 超阈值 ${OVERFITTING_GAP_THRESHOLD}(holdout ratio ${holdout.ratio}）—— 提升可能是对用例集过拟合、对 holdout 不泛化；扩充用例集或换独立外验集复核`,
+        gated: true,
+        data: { variant: worst.variant, trainScore: worst.train, holdoutScore: worst.holdout, gap: worst.gap },
+    };
+}
+/**
+ * Knowledge-gap rate above which the verdict appends an **informational** caveat.
+ * Internal-only, deliberately NOT exported: gap rate is informational, never a
+ * gate (knowledge-gap-signal-spec.md §8), so exposing this as a tunable constant
+ * would invite reading it as a pass/fail line.
+ */
+const GAP_CAVEAT_BAND = 0.2;
+/**
+ * Knowledge-gap caveat (`report.analysis.gapReports`). **Soft only — never changes
+ * the verdict level** (spec §8: a nudge, not a fail). Surfaces the treatment's gap
+ * rate with its mandatory test-set watermark (spec §7.1: a gap number without a
+ * watermark is invalid output) plus the "informational, not completeness" framing.
+ * Empty note below the band, so the common low-gap case keeps verdict headlines
+ * byte-identical.
+ */
+function gapSignalCaveat(report) {
+    const gapReports = report.analysis?.gapReports;
+    if (!gapReports)
+        return { note: '' };
+    // Worst-case (highest gap rate) over all treatments — a control-vs-many report must
+    // surface the noisiest treatment, not just variants[1]. spec §7.1: gap 数必须带
+    // test-set 水印,否则视为无效输出 —— 无水印的条目不参与(而非吐裸缺口率)。生产报告
+    // 恒带水印(report-finalize 强制),此处只防手搓 / 退化报告。
+    let gr = null;
+    for (const t of treatmentVariants(report)) {
+        const cand = gapReports[t];
+        if (!cand || cand.gapRate < GAP_CAVEAT_BAND)
+            continue;
+        if (!cand.testSetPath && !cand.testSetHash)
+            continue;
+        if (!gr || cand.gapRate > gr.gapRate)
+            gr = cand;
+    }
+    if (!gr)
+        return { note: '' };
+    const pct = (gr.gapRate * 100).toFixed(0);
+    const shortHash = gr.testSetHash ? gr.testSetHash.slice(0, 8) : '';
+    const tag = shortHash || gr.testSetPath;
+    const watermark = gr.testSetPath
+        ? `${gr.testSetPath}${shortHash ? ` @ ${shortHash}` : ''}`
+        : shortHash;
+    return {
+        note: ` · 知识缺口率 ${pct}% @ ${tag}`,
+        rationale: `知识缺口率 ${pct}%(test set: ${watermark}，N=${gr.sampleCount}）—— informational，反映当前用例集与知识库的交互、非完备性度量；高缺口提示扩充知识库或复核未覆盖文件`,
+        data: { variant: gr.variant, gapRatePct: Number(pct), testSetPath: gr.testSetPath, testSetHash: gr.testSetHash },
+    };
+}
+function recommendation(level, _perPair, lang = 'en') {
+    if (lang === 'zh') {
+        switch (level) {
+            case 'PROGRESS':
+                return '可发布 —— 实验组显著更优，且通过所有分层门控。';
+            case 'CAUTIOUS':
+                return '需排查 —— 提升是真的，但至少触发了一条告警（门控破损 / 提升微不足道 / 仅部分恢复 / 评委分歧 / 跨轮不稳 / 训练-留出过拟合）。不要盲发。';
+            case 'REGRESS':
+                return '勿发布 —— 实验组退步。检查最差的那一层，修好再重跑。';
+            case 'NOISE':
+                return '不下结论 —— 差异置信区间跨过 0，当前 N 下分辨不出效果。';
+            case 'UNDERPOWERED':
+                return '数据不足 —— 增加用例数（建议 2× 当前）后重跑。';
+            case 'SOLO':
+                return '缺对照 —— 单变体报告。用 --control baseline --treatment <名字> 重跑。';
+        }
+    }
     switch (level) {
         case 'PROGRESS':
             return 'SHIP — treatment is significantly better and passes all layer gates.';
         case 'CAUTIOUS':
-            return 'INVESTIGATE — the gain is real but at least one warning fired (broken gate, trivially small, partial recovery, judge dissent, or run-to-run unstable). Do not ship blind.';
+            return 'INVESTIGATE — the gain is real but at least one warning fired (broken gate, trivially small, partial recovery, judge dissent, run-to-run unstable, or train/holdout overfitting). Do not ship blind.';
         case 'REGRESS':
             return 'DO NOT SHIP — treatment regresses. Check the worst layer and re-run with the fix.';
         case 'NOISE':
@@ -397,22 +588,31 @@ function recommendation(level, _perPair) {
  *  spec — one verdict + four rationale bullets + one ship recommendation.
  */
 export function formatVerdictText(result, options = {}) {
+    // lang 默认 'en':保留既有英文输出逐字节不变(verdict.test 与历史 CLI 行为)。zh 只本地化
+    //  标签与 ship 建议;headline 是 Δ/CI/N 统计记号 —— 跨语言中性、且会随 report 持久化,
+    //  不翻译(翻它=改可比性锚点)。recommendation 在 format 时按 lang 重新派生,不动 computeVerdict。
+    const zh = options.lang === 'zh';
     const lines = [];
-    lines.push(`Verdict: ${result.level}`);
+    lines.push(zh ? `判定：${result.level}` : `Verdict: ${result.level}`);
     lines.push(`  ${result.headline}`);
     if (result.rationale.layerWinners)
-        lines.push(`  Layer winners: ${result.rationale.layerWinners}`);
+        lines.push(zh ? `  分层优胜：${result.rationale.layerWinners}` : `  Layer winners: ${result.rationale.layerWinners}`);
     if (result.rationale.sampleSize)
-        lines.push(`  Sample size:   ${result.rationale.sampleSize}`);
+        lines.push(zh ? `  用例规模：${result.rationale.sampleSize}` : `  Sample size:   ${result.rationale.sampleSize}`);
     if (result.rationale.stability)
-        lines.push(`  Stability:     ${result.rationale.stability}`);
+        lines.push(zh ? `  跨轮稳定：${result.rationale.stability}` : `  Stability:     ${result.rationale.stability}`);
     if (result.rationale.judgeAgreement)
-        lines.push(`  Judge α:       ${result.rationale.judgeAgreement}`);
-    if (result.rationale.shipRecommendation)
-        lines.push(`  ${result.rationale.shipRecommendation}`);
+        lines.push(zh ? `  评委 α：${result.rationale.judgeAgreement}` : `  Judge α:       ${result.rationale.judgeAgreement}`);
+    if (result.rationale.overfitting)
+        lines.push(zh ? `  过拟合：${result.rationale.overfitting}` : `  Overfitting:   ${result.rationale.overfitting}`);
+    if (result.rationale.gapSignal)
+        lines.push(zh ? `  知识缺口：${result.rationale.gapSignal}` : `  Gap signal:    ${result.rationale.gapSignal}`);
+    if (result.rationale.shipRecommendation) {
+        lines.push(`  ${zh ? recommendation(result.level, [], 'zh') : result.rationale.shipRecommendation}`);
+    }
     if (options.verbose && result.perPair && result.perPair.length > 1) {
         lines.push('');
-        lines.push('  Per-pair detail:');
+        lines.push(zh ? '  逐对明细：' : '  Per-pair detail:');
         for (const p of result.perPair) {
             lines.push(`    ${p.level}: ${p.treatment} vs ${p.control} — ${p.headline}`);
         }

package/dist/eval-workflows/batch-evaluation-workflow.d.ts

@@ -52,7 +52,7 @@ interface CompletedBatchSkillRun {
  *  导致 #183 角色误绑各存一份)。
  *  两个 variant 的 allowedSkills 都从 eval.yaml variants[].allowedSkills 取:treatment 按 skill
  *  名 entry.name 查、baseline 按保留名 `baseline` 查,挂到对应 spec 上由 prepareEvaluationRun
- *  统一绑定。baseline 的显式声明(白名单或 `[]`)必须保留——eval.yaml variant.allowedSkills
+ *  统一绑定。baseline 的显式声明(`[]`)必须保留——eval.yaml variant.allowedSkills
  *  优先于 strictBaseline 默认,漏挂会让 `--batch --config` 的 baseline 隔离配置静默失效。 */
 export declare function buildBatchVariantSpecs(entry: {
     name: string;

package/dist/eval-workflows/batch-evaluation-workflow.js

@@ -10,7 +10,7 @@ import { DEFAULT_JOBS_DIR } from '../eval-core/default-dirs.js';
  *  导致 #183 角色误绑各存一份)。
  *  两个 variant 的 allowedSkills 都从 eval.yaml variants[].allowedSkills 取:treatment 按 skill
  *  名 entry.name 查、baseline 按保留名 `baseline` 查,挂到对应 spec 上由 prepareEvaluationRun
- *  统一绑定。baseline 的显式声明(白名单或 `[]`)必须保留——eval.yaml variant.allowedSkills
+ *  统一绑定。baseline 的显式声明(`[]`)必须保留——eval.yaml variant.allowedSkills
  *  优先于 strictBaseline 默认,漏挂会让 `--batch --config` 的 baseline 隔离配置静默失效。 */
 export function buildBatchVariantSpecs(entry, variantAllowedSkills) {
     const baselineAllowed = variantAllowedSkills?.baseline;
@@ -98,7 +98,6 @@ export function buildBatchEvaluationReport({ batchRunId, skillDir, skillEntries,
         timeoutMs,
         noCache,
         dryRun: false,
-        blind: false,
         project,
         owner,
         tags,

package/dist/eval-workflows/evaluation-pipeline/report-finalize.d.ts

@@ -9,20 +9,16 @@
  *   - 可选 gapReports: 文本信号(markers / hedging)始终适用,与 tool trace 无关;
  *     带上 testSetHash 水印(spec §7.1 强制要求)
  *
- * 最后一步 `applyBlindMode` 是 blind 模式下的字段脱敏,放在所有 analysis 之后,
- * 避免脱敏后字段被分析逻辑读取。
- *
  * 仅被 orchestrator 调用;独立拆出主要为让 orchestrator 的 try-finally 主干
  * 看起来纯粹是「执行→收尾」时序。
  */
 import type { Artifact, Report, Sample, VariantResult } from '../../types/index.js';
 type EvaluationResults = Record<string, Record<string, VariantResult>>;
-export declare function finalizeEvaluationReport({ report, results, artifacts, variantNames, blind, samplesPath, samplesSourceFiles, samples, }: {
+export declare function finalizeEvaluationReport({ report, results, artifacts, variantNames, samplesPath, samplesSourceFiles, samples, }: {
     report: Report;
     results: EvaluationResults;
     artifacts: Artifact[];
     variantNames: string[];
-    blind: boolean;
     samplesPath: string;
     /** 目录模式下,bundle 内所有源文件;单文件模式下 [samplesPath]。computeTestSetHash 用。 */
     samplesSourceFiles?: string[];