oh-my-knowledge 0.38.0 → 0.40.0

package/dist/eval-core/bootstrap.d.ts

@@ -17,8 +17,14 @@
  *   - bootstrapDiffCI: CI for the difference (B - A); 0 outside the CI = significant
  *   - bootstrapWithMetric: generic interface so saturation analysis can reuse
  *
- * Reproducibility: pass a fixed `seed` to get deterministic CIs across runs.
- * Without a seed we use Math.random() — fine for production but not for tests.
+ * Reproducibility: CIs are **deterministic by default** — when no `seed` is passed,
+ * a fixed `DEFAULT_BOOTSTRAP_SEED` is used, so the same eval run twice yields
+ * byte-identical CIs (and a stable verdict near the significance boundary). This is
+ * a measurement-validity requirement: an unseeded `Math.random()` would let the
+ * `significant` flag flip between identical runs. Library callers (and specific paths
+ * such as `eval gold compare --seed`) may pass an explicit `seed` to vary the draw; the
+ * main `omk eval` deliberately exposes no seed knob — a fixed default also prevents
+ * seed-shopping for significance.
  */
 export interface BootstrapCI {
     /** Lower bound of the CI. */
@@ -42,6 +48,21 @@ export interface BootstrapDiffCI extends BootstrapCI {
 export declare const DEFAULT_BOOTSTRAP_SAMPLES = 1000;
 /** Default significance level; 0.05 → 95% CI. */
 export declare const DEFAULT_BOOTSTRAP_ALPHA = 0.05;
+/**
+ * Fixed default bootstrap seed → CIs are **deterministic by default** (omk default-strict:
+ * reproducibility affects verdict validity, so it is on by default, not opt-in). The specific
+ * value is arbitrary — only that it is **fixed** matters; it is an implementation detail, not a
+ * user-facing constant, so unlike DEFAULT_BOOTSTRAP_SAMPLES / α it is neither cited in docs nor
+ * guarded by `doc-constants-drift.test.ts`. Callers wanting a different draw pass an explicit `seed`.
+ */
+export declare const DEFAULT_BOOTSTRAP_SEED = 20260616;
+/**
+ * Confidence-level label for display: `(1 − α)·100%`. Multiple-comparison (Bonferroni) correction
+ * shrinks a pairwise α to α/K and widens the CI accordingly — the label must track α so a corrected
+ * (wider) interval is never mislabeled "95%". No alpha (single comparison / classic A-B) ⇒ nominal 95%.
+ * Shared single source for the HTML renderer and the `omk eval` CLI verdict so both read one scale.
+ */
+export declare function ciLevelLabel(alpha?: number): string;
 /**
  * Bootstrap confidence interval for the mean of a single sample.
  *
@@ -68,6 +89,36 @@ export declare function bootstrapMeanCI(scores: number[], alpha?: number, sample
  * @returns        BootstrapDiffCI with low/high of (B - A) and significant flag.
  */
 export declare function bootstrapDiffCI(scoresA: number[], scoresB: number[], alpha?: number, samples?: number, seed?: number): BootstrapDiffCI;
+/**
+ * Bootstrap CI for the *difference* of two means on **paired** observations — when A and B
+ * are two measurements of the **same unit** (e.g. control vs treatment on the same sample,
+ * or original vs alternate judge prompt on the same response). Resamples the **pair indices
+ * jointly** and averages each pair's `b - a`, so the within-pair correlation is preserved.
+ *
+ * Why paired (vs `bootstrapDiffCI`'s independent resampling): when A and B move together
+ * across units (the usual case — the same sample scored by two variants is positively
+ * correlated), much of each group's variance is shared and cancels in the per-pair diff.
+ * The independent (unpaired) bootstrap ignores that, over-states the diff's variance, and
+ * widens the CI — *conservative*, costing real power. Use paired whenever the design is
+ * paired; use `bootstrapDiffCI` only for genuinely independent groups (or where a deliberate
+ * conservative bias is wanted). The point estimate is identical (mean of per-pair diffs =
+ * difference of paired means); only the CI tightens.
+ *
+ * `significant` is derived from the **rounded** `low`/`high` (the persisted bounds), so the
+ * flag never contradicts what is stored / displayed: a CI that rounds to include 0 reads as
+ * not-significant. (Computing it on the unrounded bounds would let the JSON say `low: 0,
+ * significant: true` — a self-contradictory `CI=[0, …]` that downstream `computeVerdict` and
+ * external consumers cannot reconcile.) Matches `bootstrapDiffCI`.
+ *
+ * @param pairs    Aligned observations; `a` = control/baseline, `b` = treatment. diff = b - a.
+ * @param alpha    Significance level. Default 0.05.
+ * @param samples  Bootstrap resamples. Default 1000.
+ * @param seed     Optional seed (deterministic by default — see module header).
+ */
+export declare function bootstrapPairedDiffCI(pairs: Array<{
+    a: number;
+    b: number;
+}>, alpha?: number, samples?: number, seed?: number): BootstrapDiffCI;
 /**
  * Generic bootstrap CI for an arbitrary sample-level metric. Used by
  * saturation analysis to get CI on metrics like stddev or

package/dist/eval-core/bootstrap.js

@@ -17,8 +17,14 @@
  *   - bootstrapDiffCI: CI for the difference (B - A); 0 outside the CI = significant
  *   - bootstrapWithMetric: generic interface so saturation analysis can reuse
  *
- * Reproducibility: pass a fixed `seed` to get deterministic CIs across runs.
- * Without a seed we use Math.random() — fine for production but not for tests.
+ * Reproducibility: CIs are **deterministic by default** — when no `seed` is passed,
+ * a fixed `DEFAULT_BOOTSTRAP_SEED` is used, so the same eval run twice yields
+ * byte-identical CIs (and a stable verdict near the significance boundary). This is
+ * a measurement-validity requirement: an unseeded `Math.random()` would let the
+ * `significant` flag flip between identical runs. Library callers (and specific paths
+ * such as `eval gold compare --seed`) may pass an explicit `seed` to vary the draw; the
+ * main `omk eval` deliberately exposes no seed knob — a fixed default also prevents
+ * seed-shopping for significance.
  */
 /**
  * Default number of bootstrap resamples. Every eval path uses this unless
@@ -28,6 +34,24 @@
 export const DEFAULT_BOOTSTRAP_SAMPLES = 1000;
 /** Default significance level; 0.05 → 95% CI. */
 export const DEFAULT_BOOTSTRAP_ALPHA = 0.05;
+/**
+ * Fixed default bootstrap seed → CIs are **deterministic by default** (omk default-strict:
+ * reproducibility affects verdict validity, so it is on by default, not opt-in). The specific
+ * value is arbitrary — only that it is **fixed** matters; it is an implementation detail, not a
+ * user-facing constant, so unlike DEFAULT_BOOTSTRAP_SAMPLES / α it is neither cited in docs nor
+ * guarded by `doc-constants-drift.test.ts`. Callers wanting a different draw pass an explicit `seed`.
+ */
+export const DEFAULT_BOOTSTRAP_SEED = 20260616;
+/**
+ * Confidence-level label for display: `(1 − α)·100%`. Multiple-comparison (Bonferroni) correction
+ * shrinks a pairwise α to α/K and widens the CI accordingly — the label must track α so a corrected
+ * (wider) interval is never mislabeled "95%". No alpha (single comparison / classic A-B) ⇒ nominal 95%.
+ * Shared single source for the HTML renderer and the `omk eval` CLI verdict so both read one scale.
+ */
+export function ciLevelLabel(alpha) {
+    const pct = (1 - (alpha ?? DEFAULT_BOOTSTRAP_ALPHA)) * 100;
+    return `${Number.isInteger(pct) ? pct : Number(pct.toFixed(1))}%`;
+}
 /** Mulberry32 PRNG — seedable, deterministic for tests. */
 function mulberry32(seed) {
     let s = seed >>> 0;
@@ -40,9 +64,9 @@ function mulberry32(seed) {
     };
 }
 function makeRng(seed) {
-    if (seed == null)
-        return Math.random;
-    return mulberry32(seed);
+    // 默认确定性:无显式 seed 时退 DEFAULT_BOOTSTRAP_SEED(而非 Math.random)——否则同一 eval 两跑会得到
+    // 不同 CI,临界点 significant 翻转 → verdict 不可复现。见模块头 Reproducibility。
+    return mulberry32(seed ?? DEFAULT_BOOTSTRAP_SEED);
 }
 /** Sample n indices with replacement from [0, length) using the given PRNG. */
 function resampleIndices(length, n, rng) {
@@ -150,6 +174,59 @@ export function bootstrapDiffCI(scoresA, scoresB, alpha = DEFAULT_BOOTSTRAP_ALPH
         significant: !(low <= 0 && 0 <= high),
     };
 }
+/**
+ * Bootstrap CI for the *difference* of two means on **paired** observations — when A and B
+ * are two measurements of the **same unit** (e.g. control vs treatment on the same sample,
+ * or original vs alternate judge prompt on the same response). Resamples the **pair indices
+ * jointly** and averages each pair's `b - a`, so the within-pair correlation is preserved.
+ *
+ * Why paired (vs `bootstrapDiffCI`'s independent resampling): when A and B move together
+ * across units (the usual case — the same sample scored by two variants is positively
+ * correlated), much of each group's variance is shared and cancels in the per-pair diff.
+ * The independent (unpaired) bootstrap ignores that, over-states the diff's variance, and
+ * widens the CI — *conservative*, costing real power. Use paired whenever the design is
+ * paired; use `bootstrapDiffCI` only for genuinely independent groups (or where a deliberate
+ * conservative bias is wanted). The point estimate is identical (mean of per-pair diffs =
+ * difference of paired means); only the CI tightens.
+ *
+ * `significant` is derived from the **rounded** `low`/`high` (the persisted bounds), so the
+ * flag never contradicts what is stored / displayed: a CI that rounds to include 0 reads as
+ * not-significant. (Computing it on the unrounded bounds would let the JSON say `low: 0,
+ * significant: true` — a self-contradictory `CI=[0, …]` that downstream `computeVerdict` and
+ * external consumers cannot reconcile.) Matches `bootstrapDiffCI`.
+ *
+ * @param pairs    Aligned observations; `a` = control/baseline, `b` = treatment. diff = b - a.
+ * @param alpha    Significance level. Default 0.05.
+ * @param samples  Bootstrap resamples. Default 1000.
+ * @param seed     Optional seed (deterministic by default — see module header).
+ */
+export function bootstrapPairedDiffCI(pairs, alpha = DEFAULT_BOOTSTRAP_ALPHA, samples = DEFAULT_BOOTSTRAP_SAMPLES, seed) {
+    if (pairs.length === 0) {
+        return { low: 0, high: 0, estimate: 0, samples: 0, significant: false };
+    }
+    const n = pairs.length;
+    const diffs = pairs.map((p) => p.b - p.a);
+    const rng = makeRng(seed);
+    const resampleDiffMeans = new Array(samples);
+    for (let s = 0; s < samples; s++) {
+        const idx = resampleIndices(n, n, rng);
+        let sum = 0;
+        for (const i of idx)
+            sum += diffs[i];
+        resampleDiffMeans[s] = sum / n;
+    }
+    resampleDiffMeans.sort((a, b) => a - b);
+    const low = round4(sortedQuantile(resampleDiffMeans, alpha / 2));
+    const high = round4(sortedQuantile(resampleDiffMeans, 1 - alpha / 2));
+    return {
+        low,
+        high,
+        estimate: round4(mean(diffs)),
+        samples,
+        // significant 与持久化的(舍入)边界一致 —— 见函数头:绝不出现「low:0 但 significant:true」自相矛盾。
+        significant: !(low <= 0 && 0 <= high),
+    };
+}
 /**
  * Generic bootstrap CI for an arbitrary sample-level metric. Used by
  * saturation analysis to get CI on metrics like stddev or

package/dist/eval-core/evaluation-reporting.js

@@ -8,7 +8,7 @@ import { indexReportWrite } from './artifact-index.js';
 import { buildVariantSummary } from './schema.js';
 import { buildVariantConfig, resolveExecutionStrategy } from './execution-strategy.js';
 import { getJudgePromptHash } from '../grading/judge.js';
-import { bootstrapMeanCI, bootstrapDiffCI, DEFAULT_BOOTSTRAP_ALPHA, DEFAULT_BOOTSTRAP_SAMPLES, } from './bootstrap.js';
+import { bootstrapMeanCI, bootstrapPairedDiffCI, DEFAULT_BOOTSTRAP_ALPHA, DEFAULT_BOOTSTRAP_SAMPLES, } from './bootstrap.js';
 import { getExecutorRuntimeFingerprint } from '../executors/runtime-fingerprint.js';
 const __dirname = dirname(fileURLToPath(import.meta.url));
 function findPackageJson(startDir) {
@@ -121,6 +121,9 @@ export function aggregateReport({ runId, variants, model, judgeModel, noJudge, e
     const bootstrapSamples = request?.bootstrapSamples ?? DEFAULT_BOOTSTRAP_SAMPLES;
     let pairComparisons;
     if (bootstrapEnabled) {
+        // 不变量(见 grading/layered-scores.ts):composite 在至少一层可测时恒 ≥ 1,`compositeScore === 0`
+        // 当且仅当该样本**无任何可测层**(真·缺测,如纯评委样本且评委失败)。故 `> 0` 过滤精确剔除非测量、
+        // 绝不丢"低分内容"(评委失败已在上游当缺测,不会以 0 进 composite)。下同(control / treatment)。
         for (const variant of variants) {
             const entries = Object.values(results).map((r) => r[variant]).filter(Boolean);
             const compositeScores = entries
@@ -135,23 +138,42 @@ export function aggregateReport({ runId, variants, model, judgeModel, noJudge, e
         if (variants.length >= 2) {
             pairComparisons = [];
             const controlName = variants[0];
-            const controlEntries = Object.values(results).map((r) => r[controlName]).filter(Boolean);
-            const controlScores = controlEntries
-                .filter((e) => typeof e.compositeScore === 'number' && e.compositeScore > 0)
-                .map((e) => e.compositeScore);
+            const sampleRecords = Object.values(results);
+            // **配对** diff CI:A/B 是同一批 sample 分别过 control / treatment(配对设计),按 sample 对齐 ——
+            // 同一 sample 上 control 与 treatment 都可测(composite > 0,见上不变量)才入对。同一 sample 两 variant
+            // 的分数正相关,配对 bootstrap 据此抵消共有方差、收紧 diff CI、更有功效;旧的独立(非配对)重采样高估
+            // 方差、CI 偏宽、保守失功效(见 bootstrapPairedDiffCI)。点估计不变,只收紧 CI。
+            // 先收齐每个 treatment 的配对数据;只有 ≥2 对(否则无 CI 可算)才算一个真正被检验的比较。
+            const eligible = [];
             for (let i = 1; i < variants.length; i++) {
                 const treatmentName = variants[i];
-                const treatmentEntries = Object.values(results).map((r) => r[treatmentName]).filter(Boolean);
-                const treatmentScores = treatmentEntries
-                    .filter((e) => typeof e.compositeScore === 'number' && e.compositeScore > 0)
-                    .map((e) => e.compositeScore);
-                if (controlScores.length >= 2 && treatmentScores.length >= 2) {
-                    pairComparisons.push({
-                        control: controlName,
-                        treatment: treatmentName,
-                        diffBootstrapCI: bootstrapDiffCI(controlScores, treatmentScores, DEFAULT_BOOTSTRAP_ALPHA, bootstrapSamples),
-                    });
+                const pairs = [];
+                for (const r of sampleRecords) {
+                    const c = r[controlName];
+                    const t = r[treatmentName];
+                    const a = c && typeof c.compositeScore === 'number' && c.compositeScore > 0 ? c.compositeScore : undefined;
+                    const b = t && typeof t.compositeScore === 'number' && t.compositeScore > 0 ? t.compositeScore : undefined;
+                    if (a !== undefined && b !== undefined)
+                        pairs.push({ a, b });
                 }
+                if (pairs.length >= 2)
+                    eligible.push({ treatment: treatmentName, pairs });
+            }
+            // 多重比较(Bonferroni)校正:同时检验 K 个 treatment-vs-control 假设时,family-wise 假阳性随 K 膨胀
+            // (computeVerdict 的 worst-case roll-up 取最差 —— 任一对假阳即拉高总判定)。每对 CI 用 α/K(K = 实际
+            // 产出 CI 的比较数)把 family-wise error 压回名义 α。K=1(单 treatment / 经典 A-B)即 α 不变、与历史单对
+            // 口径逐字节一致,此时不写 alpha 字段(渲染按名义 95% CI,既有报告 / 快照不动)。CI 与 significant 同在 α/K
+            // 下算,二者自洽(绝不出现 CI 含 0 却 significant 的矛盾)。注:K 很大时 α/K 落到极端分位,1000 重采样的
+            // 尾部分位偏粗,大 K 慎读 —— 不在本 PR 提采样数。
+            const familySize = eligible.length;
+            const perComparisonAlpha = familySize >= 1 ? DEFAULT_BOOTSTRAP_ALPHA / familySize : DEFAULT_BOOTSTRAP_ALPHA;
+            for (const { treatment, pairs } of eligible) {
+                pairComparisons.push({
+                    control: controlName,
+                    treatment,
+                    diffBootstrapCI: bootstrapPairedDiffCI(pairs, perComparisonAlpha, bootstrapSamples),
+                    ...(familySize >= 2 ? { alpha: perComparisonAlpha } : {}),
+                });
             }
         }
     }

package/dist/eval-core/execution-strategy.js

@@ -66,6 +66,7 @@ export function buildVariantConfig(artifact) {
         cwd: artifact.cwd || null,
         locator: artifact.locator,
         ref: artifact.ref,
+        ...(artifact.resolvedCommit ? { resolvedCommit: artifact.resolvedCommit } : {}),
         // propagate skill-isolation declaration so report.meta.skillIsolation
         // 能在 evaluation-reporting 阶段从 variantConfigs 提取 (avoid re-resolving artifacts).
         ...(artifact.allowedSkills !== undefined && { allowedSkills: artifact.allowedSkills }),

package/dist/eval-core/schema.js

@@ -88,7 +88,8 @@ export function buildVariantResult(execResult, gradeResult, options) {
                 layeredScores.factScore = assertionFact != null
                     ? Number(((assertionFact + hardScore) / 2).toFixed(2))
                     : hardScore;
-                // Recompute composite from updated layers (保留 0 分,仅过滤真正缺失)
+                // Recompute composite from updated layers. 仅过滤 null/undefined(缺测层);评委失败时 judgeScore
+                // 本就是 undefined(见 grading 的 score=0 修复:评委失败=缺测、不以 0 进层),不存在「0 分」要保留。
                 const scores = [layeredScores.factScore, layeredScores.behaviorScore, layeredScores.judgeScore].filter((s) => s != null);
                 compositeScore = scores.length > 0 ? Number((scores.reduce((a, b) => a + b, 0) / scores.length).toFixed(2)) : compositeScore;
             }
@@ -204,8 +205,9 @@ export function buildVariantSummary(entries) {
             };
         })(),
         ...(() => {
-            // 保留 0 分用例(评委打"完全不合格"是合法低分,不是缺失)。
-            // 仅 filter null / undefined(真正缺数据,如该 sample 无对应断言或未配 judge)。
+            // 各层分量为 null/undefined = 该层缺测(无对应断言 / 未配 judge / 评委失败)。评委失败已在 grading 层
+            // 当缺测、judgeScore 留 undefined(见 score=0 修复),不存在「0 分内容」要保留;评委有效分恒 1-5。
+            // filter != null 即精确剔除缺测层。
             const factScores = ok.map((e) => e.layeredScores?.factScore).filter((s) => s != null);
             const behaviorScores = ok.map((e) => e.layeredScores?.behaviorScore).filter((s) => s != null);
             const judgeScores = ok.map((e) => e.layeredScores?.judgeScore).filter((s) => s != null);

package/dist/eval-core/verdict.d.ts

@@ -45,6 +45,16 @@ export declare const UNDERPOWERED_MIN_SAMPLES = 20;
  */
 export declare const ENSEMBLE_STRONG_PEARSON = 0.7;
 export declare const ENSEMBLE_DISSENT_PEARSON = 0.4;
+/**
+ * Run-to-run instability threshold on the median coefficient of variation (CV = stddev/mean).
+ * Once stability is **actually measured** (`--repeat ≥ 2`) and the median CV exceeds this line,
+ * a would-be PROGRESS is downgraded to CAUTIOUS — a statistically significant but run-to-run
+ * irreproducible "gain" is not shippable. It is the upper bound of the 5/15% stability bands in
+ * `docs/specs/terminology-spec.md` §5; doc ↔ code parity is guarded by
+ * `test/scripts/doc-constants-drift.test.ts`. Single-run reports (stability not measured) are
+ * never gated by it — see `computeVerdict`.
+ */
+export declare const STABILITY_UNSTABLE_CV = 0.15;
 export type VerdictLevel = 'PROGRESS' | 'CAUTIOUS' | 'REGRESS' | 'NOISE' | 'UNDERPOWERED' | 'SOLO';
 export interface VerdictResult {
     level: VerdictLevel;
@@ -85,6 +95,26 @@ export interface VerdictOptions {
  * Compute a verdict for a finished report. Pure function — no I/O.
  */
 export declare function computeVerdict(report: Report, options?: VerdictOptions): VerdictResult;
+/**
+ * Stability rationale. 三种状态:
+ *   - --repeat ≥ 2 + 有 variance 数据: 报告 CV (variation coefficient) 主指标
+ *   - --repeat ≥ 2 但 variance 缺失: 异常,标 "—" 提示数据丢失
+ *   - --repeat < 2: 显式说"未测量,需 --repeat ≥ 2",而不是默默不提
+ *
+ * 单轮场景关键:不是"稳定 = 100%"(常见误读),而是"测不到稳定性"。
+ * Verdict 必须诚实交代这个盲区,不能让用户以为没说就是 OK。
+ */
+/**
+ * 跨轮稳定性的中位 CV(variation coefficient = stddev/mean)。仅在**已测**(runs≥2)且 variance 数据齐时
+ * 返回 { runs, cv },否则 null(单轮未测 / variance 缺失 / CV 全算不出 → 不参与门控)。formatStability 的
+ * 文字、computeVerdict 的稳定性门控、renderer 的 hero CV chip 共用这一处计算,口径一致、绝不漂移。
+ * **真·中位**:偶数个 variant 取中间两项的平均(不是上中位)—— 最常见的 A/B 报告恰好是两个 variant,取上中位
+ * 会退化成"较大的那个 CV",把门控口径从「中位」悄悄变成「max」、直接改变 ship/no-ship(复审 P2)。
+ */
+export declare function medianStabilityCV(report: Report): {
+    runs: number;
+    cv: number;
+} | null;
 /**
  * Plain-text formatter for the `omk eval` verdict. Stays under 6 lines per the
  *  spec — one verdict + four rationale bullets + one ship recommendation.

package/dist/eval-core/verdict.js

@@ -28,6 +28,7 @@
  * empirical) is documented inline so users can audit and override.
  */
 import { evaluateLayerGates } from './layer-gates.js';
+import { ciLevelLabel } from './bootstrap.js';
 /**
  * Below this sample count a non-significant diff is read as UNDERPOWERED
  * (only large effects are detectable) rather than NOISE. Matches the
@@ -45,6 +46,16 @@ export const UNDERPOWERED_MIN_SAMPLES = 20;
  */
 export const ENSEMBLE_STRONG_PEARSON = 0.7;
 export const ENSEMBLE_DISSENT_PEARSON = 0.4;
+/**
+ * Run-to-run instability threshold on the median coefficient of variation (CV = stddev/mean).
+ * Once stability is **actually measured** (`--repeat ≥ 2`) and the median CV exceeds this line,
+ * a would-be PROGRESS is downgraded to CAUTIOUS — a statistically significant but run-to-run
+ * irreproducible "gain" is not shippable. It is the upper bound of the 5/15% stability bands in
+ * `docs/specs/terminology-spec.md` §5; doc ↔ code parity is guarded by
+ * `test/scripts/doc-constants-drift.test.ts`. Single-run reports (stability not measured) are
+ * never gated by it — see `computeVerdict`.
+ */
+export const STABILITY_UNSTABLE_CV = 0.15;
 /**
  * Compute a verdict for a finished report. Pure function — no I/O.
  */
@@ -89,6 +100,16 @@ export function computeVerdict(report, options = {}) {
     }
     // Single representative pair for the top-level rationale (the worst one).
     const representative = perPair.find((p) => p.level === topLevel) ?? perPair[0];
+    // 稳定性门控(报告级,非 per-pair):仅当**已测**(runs≥2)且 run-to-run 不稳(median CV > STABILITY_UNSTABLE_CV)
+    // 时,把 PROGRESS 降为 CAUTIOUS —— 显著但跨轮不可复现的"进展"不可 ship。单轮(未测稳定性)不门控:rationale
+    // 已诚实标"未测量"(terminology-spec §5「诚实交代测不到的东西」),默认单轮全降级会过激。只压 PROGRESS:已是 CAUTIOUS/REGRESS 等
+    // 不再加码,顺序与 worst-case roll-up 一致。
+    const stab = medianStabilityCV(report);
+    const stabilityGated = topLevel === 'PROGRESS' && stab !== null && stab.cv > STABILITY_UNSTABLE_CV;
+    const level = stabilityGated ? 'CAUTIOUS' : topLevel;
+    const stabilityNote = stabilityGated && stab
+        ? ` · 显著但 run-to-run 不稳(CV=${(stab.cv * 100).toFixed(1)}% > ${(STABILITY_UNSTABLE_CV * 100).toFixed(0)}%)`
+        : '';
     const significance = representative
         ? formatSignificance(representative)
         : 'no pairwise comparison available — was --bootstrap used?';
@@ -96,12 +117,12 @@ export function computeVerdict(report, options = {}) {
     const sampleSize = formatSampleSize(report);
     const stability = formatStability(report);
     const judgeAgreement = formatJudgeAgreement(report);
-    const shipRecommendation = recommendation(topLevel, perPair);
+    const shipRecommendation = recommendation(level, perPair);
     return {
-        level: topLevel,
+        level,
         headline: representative
-            ? `${topLevel} · ${representative.treatment} vs ${representative.control}: ${representative.headline}`
-            : `${topLevel} · ${variants.length} variants`,
+            ? `${level} · ${representative.treatment} vs ${representative.control}: ${representative.headline}${stabilityNote}`
+            : `${level} · ${variants.length} variants`,
         perPair,
         rationale: {
             significance,
@@ -130,7 +151,10 @@ function verdictForPair(pair, summary, sampleCount, report, gateThreshold, trivi
     // Layer-gate check: did any layer fall below threshold for either variant?
     const cGate = evaluateLayerGates({ [control]: summary[control] }, gateThreshold);
     const tGate = evaluateLayerGates({ [treatment]: summary[treatment] }, gateThreshold);
-    // No bootstrap CI available → fall back to point-estimate diff comparison.
+    // No bootstrap CI available → fall back to point-estimate diff comparison. 这是 `--no-bootstrap` 的**降级
+    // 模式**:bootstrap 默认开,正常路径永远有 CI,这里只在用户显式关掉时触达。降级路径刻意不对称——正 Δ 最多给
+    // CAUTIOUS(没 CI 不敢判 PROGRESS),负 Δ 直接 REGRESS(不做显著性检验)。这对"检测变差"是保守安全方向(宁可
+    // 误报回归也别漏掉),代价是把噪声级的负 Δ 也叫 REGRESS;要严谨结论就别关 bootstrap。
     if (!diff) {
         const cMean = avgComposite(summary[control]);
         const tMean = avgComposite(summary[treatment]);
@@ -146,7 +170,10 @@ function verdictForPair(pair, summary, sampleCount, report, gateThreshold, trivi
             headline,
         };
     }
-    const headlineCore = `Δ=${diff.estimate >= 0 ? '+' : ''}${diff.estimate} CI=[${diff.low}, ${diff.high}]`;
+    // 多重比较把本对的 α 收到 α/K → CI 变宽,标签随 α 走(与 HTML pairwise 表同口径);K=1(无 alpha)不加标签,
+    // headline 与历史逐字节一致。让 omk eval CLI 也诚实显示真实置信水平,而非裸区间。
+    const ciLabel = pair.alpha != null ? `${ciLevelLabel(pair.alpha)} ` : '';
+    const headlineCore = `Δ=${diff.estimate >= 0 ? '+' : ''}${diff.estimate} ${ciLabel}CI=[${diff.low}, ${diff.high}]`;
     if (!diff.significant) {
         // Diff CI contains 0. Distinguish "underpowered (saturation says: more samples needed)"
         // from "noise (saturation says: we're saturated, the effect just isn't there)".
@@ -233,7 +260,10 @@ function avgComposite(s) {
         return 0;
     if (typeof s.avgCompositeScore === 'number')
         return s.avgCompositeScore;
-    // Fallback: average of the three layers when composite isn't on the summary.
+    // 兜底:summary 无 avgCompositeScore 时,用三层均值近似。**有偏**:真 composite 是 per-sample
+    // (present-layers 均值)再跨 sample 平均;这里是「跨 sample 的层均值」再跨层平均,各层在不同 sample 上缺失
+    // 不均时两者不等(Jensen / 分母不一致)。仅 no-bootstrap 降级路径 + summary 缺 composite 的老报告才触达
+    // (默认开 bootstrap、新报告必带 avgCompositeScore,故极罕见),不值得回填 per-sample 重算 —— 标注保留近似。
     const layers = [s.avgFactScore, s.avgBehaviorScore, s.avgJudgeScore].filter((x) => typeof x === 'number');
     if (layers.length === 0)
         return 0;
@@ -289,16 +319,20 @@ function formatSampleSize(report) {
  * 单轮场景关键:不是"稳定 = 100%"(常见误读),而是"测不到稳定性"。
  * Verdict 必须诚实交代这个盲区,不能让用户以为没说就是 OK。
  */
-function formatStability(report) {
+/**
+ * 跨轮稳定性的中位 CV(variation coefficient = stddev/mean)。仅在**已测**(runs≥2)且 variance 数据齐时
+ * 返回 { runs, cv },否则 null(单轮未测 / variance 缺失 / CV 全算不出 → 不参与门控)。formatStability 的
+ * 文字、computeVerdict 的稳定性门控、renderer 的 hero CV chip 共用这一处计算,口径一致、绝不漂移。
+ * **真·中位**:偶数个 variant 取中间两项的平均(不是上中位)—— 最常见的 A/B 报告恰好是两个 variant,取上中位
+ * 会退化成"较大的那个 CV",把门控口径从「中位」悄悄变成「max」、直接改变 ship/no-ship(复审 P2)。
+ */
+export function medianStabilityCV(report) {
     const runs = report.variance?.runs ?? report.meta?.request?.repeat ?? 1;
-    if (runs < 2) {
-        return '稳定性未测量(单轮评测,需 --repeat ≥ 2 才能测 CV)';
-    }
+    if (runs < 2)
+        return null;
     const variance = report.variance?.perVariant;
-    if (!variance || Object.keys(variance).length === 0) {
-        return `runs=${runs} 但 variance 数据缺失`;
-    }
-    // CV = stddev / mean,取所有 variant 的中位数(单 variant 易有 NaN/0,聚合更稳)
+    if (!variance || Object.keys(variance).length === 0)
+        return null;
     const cvs = [];
     for (const v of Object.values(variance)) {
         if (typeof v.stddev === 'number' && typeof v.mean === 'number' && v.mean > 0) {
@@ -306,12 +340,28 @@ function formatStability(report) {
         }
     }
     if (cvs.length === 0)
-        return `runs=${runs}, CV 计算失败(stddev/mean 数据缺失)`;
+        return null;
     cvs.sort((a, b) => a - b);
-    const median = cvs[Math.floor(cvs.length / 2)];
-    const cvPct = (median * 100).toFixed(1);
-    // 阈值参考 terminology-spec §5:<5% 稳 / 5-15% 中 / >15% 不稳
-    const verdict = median < 0.05 ? '稳定' : median < 0.15 ? '中等' : '不稳';
+    const mid = Math.floor(cvs.length / 2);
+    const median = cvs.length % 2 === 0 ? (cvs[mid - 1] + cvs[mid]) / 2 : cvs[mid];
+    return { runs, cv: median };
+}
+function formatStability(report) {
+    const runs = report.variance?.runs ?? report.meta?.request?.repeat ?? 1;
+    if (runs < 2) {
+        return '稳定性未测量(单轮评测,需 --repeat ≥ 2 才能测 CV)';
+    }
+    const variance = report.variance?.perVariant;
+    if (!variance || Object.keys(variance).length === 0) {
+        return `runs=${runs} 但 variance 数据缺失`;
+    }
+    const stab = medianStabilityCV(report);
+    if (!stab)
+        return `runs=${runs}, CV 计算失败(stddev/mean 数据缺失)`;
+    const cvPct = (stab.cv * 100).toFixed(1);
+    // 阈值参考 terminology-spec §5:<5% 稳 / 5-15% 中 / >15% 不稳。不稳上界 = STABILITY_UNSTABLE_CV,
+    // 与门控同一根线:label 判"不稳" ⟺ 门控触发(cv > STABILITY_UNSTABLE_CV)。
+    const verdict = stab.cv < 0.05 ? '稳定' : stab.cv <= STABILITY_UNSTABLE_CV ? '中等' : '不稳';
     return `CV=${cvPct}% (${verdict}, runs=${runs}; 阈值 <5%=稳/5-15%=中/>15%=不稳)`;
 }
 function formatJudgeAgreement(report) {
@@ -331,7 +381,7 @@ function recommendation(level, _perPair) {
         case 'PROGRESS':
             return 'SHIP — treatment is significantly better and passes all layer gates.';
         case 'CAUTIOUS':
-            return 'INVESTIGATE — the gain is real but at least one warning fired (broken gate, trivially small, or partial recovery). Do not ship blind.';
+            return 'INVESTIGATE — the gain is real but at least one warning fired (broken gate, trivially small, partial recovery, judge dissent, or run-to-run unstable). Do not ship blind.';
         case 'REGRESS':
             return 'DO NOT SHIP — treatment regresses. Check the worst layer and re-run with the fix.';
         case 'NOISE':

package/dist/grading/assertions.js

@@ -1,5 +1,6 @@
 import { resolve } from 'node:path';
 import _Ajv from 'ajv';
+import { ASSERTION_LAYER } from './layered-scores.js';
 const Ajv = _Ajv.default ?? _Ajv;
 const ajv = new Ajv();
 const CUSTOM_ASSERTION_TIMEOUT_MS = 30_000;
@@ -266,6 +267,30 @@ function evalAssertion(output, assertion, ctx) {
             return false;
     }
 }
+/**
+ * assert-set 是布尔组合器,不是叶子断言,没有静态的「层」。它在 runAssertions 里只产出一条聚合明细
+ * (`type: 'assert-set'`、aggregate pass/fail),computeLayeredScores 无法据此判 fact / behavior。
+ * 这里在 grading 期(能看到 children 树时)按**叶子子断言**的层解析:递归收集所有叶子(穿透嵌套 assert-set),
+ *   - 同层(全 fact 或全 behavior)→ 返回该层:聚合 pass/fail 归这一层,与 any/all 模式无关(同层信号无歧义)。
+ *   - 混层 / 空 / 含未分类叶子 → 返回 undefined:不归层(混层组合器塞进单层会引入口径偏差,诚实做法是不计入分层)。
+ * 解析结果落到 detail.layer,computeLayeredScores 优先读它。
+ */
+function resolveAssertSetLayer(assertion) {
+    const layers = new Set();
+    const collect = (a) => {
+        if (a.type === 'assert-set') {
+            for (const c of a.children ?? [])
+                collect(c);
+            return;
+        }
+        layers.add(ASSERTION_LAYER[a.type] ?? 'unknown');
+    };
+    collect(assertion);
+    if (layers.size !== 1)
+        return undefined; // 0 / 混层 → 不归层
+    const [only] = [...layers];
+    return only === 'unknown' ? undefined : only;
+}
 export function runAssertions(output, assertions, context = {}) {
     const outputLower = output.toLowerCase();
     const toolCalls = context.toolCalls || [];
@@ -276,11 +301,14 @@ export function runAssertions(output, assertions, context = {}) {
         const weight = assertion.weight ?? 1;
         const raw = evalAssertion(output, assertion, ctx);
         const passed = assertion.not ? !raw : raw;
+        // assert-set 组合器:在此(能看到 children)解析其层,供 computeLayeredScores 用;叶子断言按静态映射归层、不带 layer。
+        const layer = assertion.type === 'assert-set' ? resolveAssertSetLayer(assertion) : undefined;
         details.push({
             type: assertion.type,
             value: assertion.value ?? assertion.pattern ?? assertion.values?.join(', ') ?? '',
             weight,
             passed,
+            ...(layer ? { layer } : {}),
         });
     }
     const totalWeight = details.reduce((s, d) => s + d.weight, 0);

package/dist/grading/debias-validate.js

@@ -24,7 +24,7 @@
  * large/expensive evaluations can opt in deliberately.
  */
 import { llmJudge } from './judge.js';
-import { bootstrapDiffCI } from '../eval-core/bootstrap.js';
+import { bootstrapPairedDiffCI } from '../eval-core/bootstrap.js';
 /**
  * Map a bootstrap diff CI to a verdict bucket. The ranges are deliberately
  * conservative: we only label "strong" when the CI fully sits >= |0.5| away
@@ -119,7 +119,11 @@ export async function validateLengthDebias(input) {
     }
     const meanOriginal = avg(pairs.map((p) => p.originalScore));
     const meanAlternate = avg(pairs.map((p) => p.alternateScore));
-    const diffCI = bootstrapDiffCI(pairs.map((p) => p.originalScore), pairs.map((p) => p.alternateScore), 0.05, bootstrapSamples, seed);
+    // **配对** diff CI:每个 sample 同时有 original 与 alternate prompt 两个分数(同一回答、两个 judge prompt
+    // = 配对设计)。原先拆成两数组喂独立重采样,丢弃了配对、高估方差、CI 偏宽 —— 对一个**检测**长度偏置敏感性
+    // 的工具,保守方向恰好是错的(更难检出真实偏置)。改配对:重采样 sample 下标、按 (alternate − original) 算,
+    // 保留 within-sample 相关、收紧 CI,提升对偏置的检出力。diff = b − a = alternate − original(同原约定)。
+    const diffCI = bootstrapPairedDiffCI(pairs.map((p) => ({ a: p.originalScore, b: p.alternateScore })), 0.05, bootstrapSamples, seed);
     const verdict = classifyVerdict(diffCI);
     return {
         variant,

package/dist/grading/index.js

@@ -109,7 +109,12 @@ export async function grade({ output, sample, judgeModels, judgeExecutors, allow
         const judge = useEnsemble
             ? await llmJudgeEnsemble(rubricOptions, judgeModels, executorByName, judgeRepeat)
             : await llmJudgeRepeat(rubricOptions, judgeRepeat);
-        results.llmScore = judge.score;
+        // judge.score <= 0 表示该样本所有判定尝试都失败(非 JSON / parse / executor 错——见 judge.ts 失败哨兵
+        // 268/275/298,prompt 只发 1-5),是**缺测**而非「0 分内容」。不写 llmScore(留 undefined)→ 让
+        // computeLayeredScores 当缺层排除,而不是把基础设施失败当内容分污染 composite(与多维度路径 filter(s>0)
+        // 及多轮路径 validSamples 已排除失败同口径)。llmReason / scoreSamples 仍记录,供诊断"为何失败"。
+        if (judge.score > 0)
+            results.llmScore = judge.score;
         results.llmReason = judge.reason;
         if (judge.reasoning)
             results.llmReasoning = judge.reasoning;

package/dist/grading/layered-scores.d.ts

@@ -1,4 +1,19 @@
 import type { AssertionDetail, LayeredScores } from '../types/index.js';
+/**
+ * 每个 assertion 类型归到事实层 / 行为层。**单一来源**:`computeLayeredScores` 据此把 assertion 明细拆进
+ * fact / behavior 两层算分。
+ *   - **事实层(fact)**:测输出内容对不对 —— 命中 / 匹配 / 结构合法 / 与参考的文本相似度。
+ *   - **行为层(behavior)**:测做事的方式 —— 长度 / 成本 / 轮次 / 工具调用 / 必经里程碑,不看内容本身。
+ *
+ * 不变量:**runner 支持的每个 assertion 类型都必须能被归层**,否则该类型的 pass/fail 会被 computeLayeredScores
+ * 从 fact 与 behavior 同时漏掉 —— 既不报错也不进 composite,静默丢分(曾漏掉七类:mock_hit / rouge_n_min /
+ * bleu_min / levenshtein_max + RAG 三件套 faithfulness / answer_relevancy / context_recall)。叶子断言在此静态
+ * 分类;组合器 `assert-set` 没有静态层,由 `assertions.ts` 的 resolveAssertSetLayer 在 grading 期按其叶子 children
+ * 解析(同层→归层、混层→不计),结果落 detail.layer。`test/grading/layered-scores-exhaustiveness.test.ts` 扫
+ * runner 源(evalAssertion 的 case ∪ `assertion.type ===` 组合器 ∪ ASYNC_ASSERTION_TYPES)守住:新增类型既不在本
+ * 映射、又不是已知组合器,即 CI 失败。
+ */
+export declare const ASSERTION_LAYER: Record<string, 'fact' | 'behavior'>;
 interface CompositeInput {
     assertions?: {
         details?: AssertionDetail[];