oh-my-knowledge 0.39.0 → 0.40.0

package/dist/assets/agent-skills/omk/references/commands.md

@@ -403,6 +403,7 @@ omk observe [sessionsDir] [flags]
 
 **Flags:**
 
+- `--feedback` `boolean`:把生产健康观测反哺已纳管的同名 skill（--no-feedback 关闭）
 - `--from` `option`:起始时间 ISO，优先级高于 --last
 - `--global` `boolean`:写全局 ~/.oh-my-knowledge/observe-health，而非项目 .omk/observe-health
 - `--kb` `option`:知识库 root，启用 KB-aware 分析

package/dist/authoring/evolver.js

@@ -868,11 +868,15 @@ export async function evolveSkill({ skillPath, samplesPath, rounds = 5, target =
         // Significance accept gate: accept only when the candidate is *significantly*
         // above the current best on the decision (val) set, not merely numerically higher
         // — rejecting gains indistinguishable from judge noise. `lastReport` is the current
-        // best's fresh eval and `candidateReport` the candidate's, over the same samples;
-        // bootstrapDiffCI resamples the two arrays independently (conservative — not a paired
-        // bootstrap). Under-powered decision sets degrade to the legacy point-estimate accept
-        // (note: that path compares the prior-round best scalar, not this fresh re-eval) and
-        // flag `gate.underpowered`.
+        // best's fresh eval and `candidateReport` the candidate's, over the same samples.
+        // **Deliberately UNPAIRED here** (independent resampling, `decideAccept` → `bootstrapDiffCI`),
+        // unlike the report verdict / debias which switched to `bootstrapPairedDiffCI`: this is an
+        // *optimization accept-gate*, and the unpaired bootstrap's wider, conservative CI is the
+        // wanted bias — it raises the bar to accept, so evolve does not chase paired-tightened,
+        // marginally-significant gains that risk overfitting the decision set. Power is not the goal
+        // for de-/escalation gates; not accepting noise is. Under-powered decision sets degrade to the
+        // legacy point-estimate accept (note: that path compares the prior-round best scalar, not this
+        // fresh re-eval) and flag `gate.underpowered`.
         const valIds = split ? split.valIds : new Set(allSampleIds);
         const bestScores = perSampleComposite(lastReport, lastVariantKey, valIds);
         const candScores = perSampleComposite(candidateReport, candidateVariantKey, valIds);

package/dist/cli/commands/list.d.ts

@@ -2,7 +2,10 @@ import { BaseCommand } from '../oclif/base-command.js';
 import { sanitizeCell } from '../lib/cell-format.js';
 import { type ManagedListRow } from '../../managed/index.js';
 import type { CliLang } from '../lib/i18n.js';
-/** CJK 全角字符按 2 列计宽,使含中文表头的列也能对齐。 */
+/** CJK 全角字符 + 星平面 emoji 按 2 列计宽,使含中文表头 / emoji 标记(如 🔬)的列也能对齐。
+ *  逐**码点**迭代(for...of),星平面 emoji 是单码点,加 `\u{1F300}-\u{1FAFF}`(含 🔬 U+1F52C)判 2 列。
+ *  注:`⚠️` 是「U+26A0 + U+FE0F」两码点、各 1 列合计 2,本就与 2 列渲染对齐,不另判(避免重复计数);
+ *  `✓`(U+2713)终端按 1 列渲染、保持 1。 */
 export declare function dispWidth(s: string): number;
 /** 按**显示宽度**截断(不是 code unit):逐码点累加 dispWidth,绝不切断 surrogate 对、CJK 也不溢出列。 */
 export declare function truncate(s: string, max: number): string;

package/dist/cli/commands/list.js

@@ -4,11 +4,14 @@ import { BaseCommand } from '../oclif/base-command.js';
 import { tCli } from '../lib/i18n.js';
 import { sanitizeCell } from '../lib/cell-format.js';
 import { globalManagedDir, listManagedRows, managedDir, resolveManagedDir, } from '../../managed/index.js';
-/** CJK 全角字符按 2 列计宽,使含中文表头的列也能对齐。 */
+/** CJK 全角字符 + 星平面 emoji 按 2 列计宽,使含中文表头 / emoji 标记(如 🔬)的列也能对齐。
+ *  逐**码点**迭代(for...of),星平面 emoji 是单码点,加 `\u{1F300}-\u{1FAFF}`(含 🔬 U+1F52C)判 2 列。
+ *  注:`⚠️` 是「U+26A0 + U+FE0F」两码点、各 1 列合计 2,本就与 2 列渲染对齐,不另判(避免重复计数);
+ *  `✓`(U+2713)终端按 1 列渲染、保持 1。 */
 export function dispWidth(s) {
     let w = 0;
     for (const ch of s)
-        w += /[ᄀ-ᅟ⺀-꓏가-힣豈-﫿︰-﹏＀-｠￠-￦]/.test(ch) ? 2 : 1;
+        w += /[ᄀ-ᅟ⺀-꓏가-힣豈-﫿︰-﹏＀-｠￠-￦\u{1F300}-\u{1FAFF}]/u.test(ch) ? 2 : 1;
     return w;
 }
 function pad(s, width) {
@@ -45,7 +48,9 @@ export function renderTable(rows, lang) {
         truncate(sanitizeCell(r.name), 40), // name 与 source 同为用户可控、可超长 → 同样按显示宽度截断,防撑爆表宽
         r.kind, // ArtifactKind 枚举(validator 已收窄),无需洗
         // 不可达 → 标「?」(drift 未核),绝不冒充 stale;reachable 且漂移才 stale ⚠️;已人工接受标 promoted ✓。
-        !r.reachable ? `${r.state} ?` : r.state === 'stale' ? 'stale ⚠️' : r.state === 'promoted' ? 'promoted ✓' : r.state,
+        // 生产盲区 🔬 与生命周期**正交**(observe 量线上部署版),叠加在 state token 之后,不替换 state。
+        (!r.reachable ? `${r.state} ?` : r.state === 'stale' ? 'stale ⚠️' : r.state === 'promoted' ? 'promoted ✓' : r.state)
+            + (r.productionGap ? ' 🔬' : ''),
         r.latestVerdict ? sanitizeCell(r.latestVerdict) : '—',
         `${r.currentEvidenceCount}/${r.totalEvidenceCount}`,
         truncate(sanitizeCell(r.sourceLabel), 48),
@@ -99,7 +104,8 @@ export default class List extends BaseCommand {
             const hasDrift = rows.some((r) => r.drifted);
             const hasUnreachable = rows.some((r) => !r.reachable);
             const hasPromoted = rows.some((r) => r.state === 'promoted');
-            if (hasDrift || hasUnreachable || hasPromoted)
+            const hasProductionGap = rows.some((r) => r.productionGap);
+            if (hasDrift || hasUnreachable || hasPromoted || hasProductionGap)
                 process.stderr.write('\n');
             if (hasPromoted)
                 process.stderr.write(tCli('cli.list.promoted_note', lang));
@@ -107,6 +113,8 @@ export default class List extends BaseCommand {
                 process.stderr.write(tCli('cli.list.drift_note', lang));
             if (hasUnreachable)
                 process.stderr.write(tCli('cli.list.unreachable_note', lang));
+            if (hasProductionGap)
+                process.stderr.write(tCli('cli.list.production_gap_note', lang));
             process.stderr.write(tCli('cli.list.legend', lang));
         });
     }

package/dist/cli/commands/observe/index.d.ts

@@ -9,6 +9,14 @@ export declare function persistObserveHealthReport(report: SkillHealthReport, ou
     id: string;
     jsonPath: string;
 };
+/**
+ * SkillHealthReport → managed 反哺的结构化最小入参(#235)。纯映射、可单测 —— 把「observe 报告 →
+ * ObserveReportView」这段层间胶水从 CLI 副作用里拆出来,免得 healthBand 取错字段 / observedAt 取错时刻
+ * 这类映射 bug 无人验。`observedAt` 取**流量窗口结束时刻**(timeRange.to,空则退 generatedAt),不是「此刻」
+ * 的 generatedAt —— 否则 latest-wins 会把所有观测当成一样新(见 ManagedObservation.observedAt)。
+ * `healthBand` 由 observability 的 `healthBandOf` 逐 skill 算(阈值单一来源,注入以保可测)。
+ */
+export declare function buildObserveReportView(report: SkillHealthReport, reportId: string, healthBandOf: (weightedGapRate: number) => 'green' | 'yellow' | 'red'): import('../../../managed/index.js').ObserveReportView;
 export default class Observe extends BaseCommand {
     static description: string;
     static examples: {
@@ -27,6 +35,7 @@ export default class Observe extends BaseCommand {
         skills: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
         'output-dir': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
         global: import("@oclif/core/interfaces").BooleanFlag<boolean>;
+        feedback: import("@oclif/core/interfaces").BooleanFlag<boolean>;
     };
     run(): Promise<void>;
 }

package/dist/cli/commands/observe/index.js

@@ -23,6 +23,68 @@ export function persistObserveHealthReport(report, outDir) {
     indexObserveWrite(report, jsonPath, outDir, id);
     return { id, jsonPath };
 }
+// 盲区信号类型 → 人话标签(建议补样本提示用)。技术枚举键的展示名,zh/en 分列。
+const GAP_AREA_LABELS = {
+    failed_search: { zh: '检索失败', en: 'failed search' },
+    explicit_marker: { zh: '显式缺口', en: 'explicit gap' },
+    hedging: { zh: '含糊回避', en: 'hedging' },
+    repeated_failure: { zh: '反复失败', en: 'repeated failure' },
+};
+/** 取盲区计数最高的前几类,组成「建议补哪类用例」的人话区域串(只展示信号所在,不生成具体用例)。
+ *  只迭代**四个已知盲区类型**(GAP_AREA_LABELS 的键),记录 / 报告里若混入额外键一律忽略,不进展示。 */
+function topGapAreas(gapByType, lang) {
+    const sep = lang === 'zh' ? '、' : ', ';
+    const areas = Object.keys(GAP_AREA_LABELS)
+        .map((k) => [k, gapByType[k] ?? 0])
+        .filter(([, n]) => n > 0)
+        .sort((a, b) => b[1] - a[1])
+        .slice(0, 3)
+        .map(([k]) => GAP_AREA_LABELS[k][lang])
+        .join(sep);
+    return areas || (lang === 'zh' ? '未归类盲区' : 'uncategorized gaps');
+}
+/**
+ * SkillHealthReport → managed 反哺的结构化最小入参(#235)。纯映射、可单测 —— 把「observe 报告 →
+ * ObserveReportView」这段层间胶水从 CLI 副作用里拆出来,免得 healthBand 取错字段 / observedAt 取错时刻
+ * 这类映射 bug 无人验。`observedAt` 取**流量窗口结束时刻**(timeRange.to,空则退 generatedAt),不是「此刻」
+ * 的 generatedAt —— 否则 latest-wins 会把所有观测当成一样新(见 ManagedObservation.observedAt)。
+ * `healthBand` 由 observability 的 `healthBandOf` 逐 skill 算(阈值单一来源,注入以保可测)。
+ */
+export function buildObserveReportView(report, reportId, healthBandOf) {
+    return {
+        reportId,
+        observedAt: report.meta.timeRange?.to || report.meta.generatedAt,
+        skills: Object.values(report.bySkill).map((s) => ({
+            skillName: s.skillName,
+            segmentCount: s.segmentCount,
+            gapRate: s.gap.gapRate,
+            weightedGapRate: s.gap.weightedGapRate,
+            confidence: s.confidence,
+            healthBand: healthBandOf(s.gap.weightedGapRate),
+            gapByType: s.gap.byType,
+        })),
+    };
+}
+/**
+ * observe → 管理支柱反哺(#235):把每个 skill 的生产健康落成观测追加进同名受管记录,并打印「已记录 / 生产
+ * 盲区警示」。**非致命**:管理是 observe 旁路,任何异常都不该让 observe 失败(try/catch 吞掉)。observability /
+ * managed 运行时函数动态 import,与 observe 主体一致、不拖累 CLI 启动。
+ */
+async function recordObserveFeedback(report, reportId, lang) {
+    try {
+        const { healthBandOf } = await import('../../../observability/skill-health-analyzer.js');
+        const { recordObserveHealth } = await import('../../../managed/index.js');
+        const written = recordObserveHealth(buildObserveReportView(report, reportId, healthBandOf));
+        for (const w of written) {
+            process.stdout.write(w.isProductionGap
+                ? tCli('cli.observe.production_gap', lang, { name: w.name, areas: topGapAreas(w.gapByType, lang) })
+                : tCli('cli.observe.observation_recorded', lang, { name: w.name }));
+        }
+    }
+    catch {
+        // 反哺是 observe 旁路,任何异常都不该让 observe 失败。
+    }
+}
 // `omk observe <sessions-dir>` 是默认命令 —— 分析 sessions 目录的 skill 调用健康度，产出 observe-health 报告(JSON)，
 // 由 Studio 健康报告页按需渲染。observe 这条线的另一条产物是观测收件箱(observe-inbox)，走子命令 ingest / inbox / show。
 export default class Observe extends BaseCommand {
@@ -80,6 +142,14 @@ export default class Observe extends BaseCommand {
                 en: 'Write to global ~/.oh-my-knowledge/observe-health instead of project .omk/observe-health',
             }),
         }),
+        feedback: Flags.boolean({
+            default: true,
+            allowNo: true,
+            description: bilingual({
+                zh: '把生产健康观测反哺已纳管的同名 skill（--no-feedback 关闭）',
+                en: 'Feed production-health observations back to managed skills of the same name (--no-feedback to disable)',
+            }),
+        }),
     };
     async run() {
         const { args, flags } = await this.parse(Observe);
@@ -120,7 +190,7 @@ export default class Observe extends BaseCommand {
             const outDir = flags['output-dir']
                 ? resolve(flags['output-dir'])
                 : (flags.global ? globalObserveHealthDir() : projectObserveHealthDir());
-            const { jsonPath } = persistObserveHealthReport(report, outDir);
+            const { id, jsonPath } = persistObserveHealthReport(report, outDir);
             const { sessionCount, segmentCount, toolCallCount, toolFailureRate } = report.meta;
             console.log('');
             console.log(`sessions: ${sessionCount} · segments: ${segmentCount} · tool calls: ${toolCallCount} · fail rate: ${(toolFailureRate * 100).toFixed(1)}%`);
@@ -139,6 +209,9 @@ export default class Observe extends BaseCommand {
             console.log('');
             console.log(`report written to: ${jsonPath}`);
             console.log(tCli('cli.observe.view_hint', lang));
+            // #235 受管反哺:把生产健康观测落进同名受管 skill(--no-feedback 关)。非致命旁路。
+            if (flags.feedback)
+                await recordObserveFeedback(report, id, lang);
         });
     }
 }

package/dist/cli/lib/i18n-dict/common.d.ts

@@ -1,3 +1,3 @@
 import type { CliMessage } from './types.js';
-export type CommonMessageKey = 'cli.common.unknown_domain' | 'cli.common.error_prefix' | 'cli.common.skill_dir_not_found' | 'cli.common.skill_file_not_found' | 'cli.common.skill_dir_no_skill_md' | 'cli.common.report_not_found' | 'cli.common.no_judge_model' | 'cli.common.judge_models_single_only' | 'cli.common.warn_load_samples_failed' | 'cli.update.new_version_available' | 'cli.update.box_title' | 'cli.update.box_version_line' | 'cli.update.box_upgrade_line' | 'cli.update.box_silence_line' | 'cli.observe.view_hint' | 'cli.studio.started' | 'cli.studio.stop_hint' | 'cli.studio.open_failed' | 'cli.doctor.no_skill_found' | 'cli.doctor.samples_detected' | 'cli.doctor.progress_skill_start' | 'cli.doctor.progress_skill_done';
+export type CommonMessageKey = 'cli.common.unknown_domain' | 'cli.common.error_prefix' | 'cli.common.skill_dir_not_found' | 'cli.common.skill_file_not_found' | 'cli.common.skill_dir_no_skill_md' | 'cli.common.report_not_found' | 'cli.common.no_judge_model' | 'cli.common.judge_models_single_only' | 'cli.common.warn_load_samples_failed' | 'cli.update.new_version_available' | 'cli.update.box_title' | 'cli.update.box_version_line' | 'cli.update.box_upgrade_line' | 'cli.update.box_silence_line' | 'cli.observe.view_hint' | 'cli.observe.observation_recorded' | 'cli.observe.production_gap' | 'cli.studio.started' | 'cli.studio.stop_hint' | 'cli.studio.open_failed' | 'cli.doctor.no_skill_found' | 'cli.doctor.samples_detected' | 'cli.doctor.progress_skill_start' | 'cli.doctor.progress_skill_done';
 export declare const commonDict: Record<CommonMessageKey, CliMessage>;

package/dist/cli/lib/i18n-dict/common.js

@@ -59,6 +59,14 @@ export const commonDict = {
         zh: '分析 JSON 已写入 output-dir；后续可用 omk observe 持续生成健康报告。',
         en: 'Analysis JSON written to output-dir; use omk observe to keep producing health reports.',
     },
+    'cli.observe.observation_recorded': {
+        zh: '🔭 已为受管 skill「{name}」记录生产健康观测。\n',
+        en: '🔭 Recorded a production-health observation for managed skill "{name}".\n',
+    },
+    'cli.observe.production_gap': {
+        zh: '⚠️ 受管 skill「{name}」线上检测到生产盲区（{areas}）。建议补充对应用例后重跑 omk eval —— observe 只提示，不改样本集。\n',
+        en: '⚠️ Managed skill "{name}" shows production gaps in real traffic ({areas}). Consider adding matching samples and re-running omk eval — observe only suggests, it does not edit the sample set.\n',
+    },
     'cli.studio.started': {
         zh: 'studio 已启动：{url}',
         en: 'Studio running at {url}',

package/dist/cli/lib/i18n-dict/list.d.ts

@@ -1,3 +1,3 @@
 import type { CliMessage } from './types.js';
-export type ListMessageKey = 'cli.list.header' | 'cli.list.empty' | 'cli.list.empty_hint' | 'cli.list.col_name' | 'cli.list.col_kind' | 'cli.list.col_state' | 'cli.list.col_verdict' | 'cli.list.col_evidence' | 'cli.list.col_source' | 'cli.list.drift_note' | 'cli.list.unreachable_note' | 'cli.list.promoted_note' | 'cli.list.legend';
+export type ListMessageKey = 'cli.list.header' | 'cli.list.empty' | 'cli.list.empty_hint' | 'cli.list.col_name' | 'cli.list.col_kind' | 'cli.list.col_state' | 'cli.list.col_verdict' | 'cli.list.col_evidence' | 'cli.list.col_source' | 'cli.list.drift_note' | 'cli.list.unreachable_note' | 'cli.list.promoted_note' | 'cli.list.production_gap_note' | 'cli.list.legend';
 export declare const listDict: Record<ListMessageKey, CliMessage>;

package/dist/cli/lib/i18n-dict/list.js

@@ -29,6 +29,10 @@ export const listDict = {
         zh: '✓ = 当前版本已按证据人工接受为 promoted（omk promote）。\n',
         en: '✓ = current version accepted as promoted on evidence (omk promote).\n',
     },
+    'cli.list.production_gap_note': {
+        zh: '🔬 = observe 在线上检测到生产盲区（与生命周期无关的版本无关信号）；建议补对应用例后重跑 omk eval。\n',
+        en: '🔬 = observe detected a production gap in real traffic (a version-agnostic signal, orthogonal to lifecycle); add matching samples and re-run omk eval.\n',
+    },
     'cli.list.legend': {
         zh: '证据列 = 当前有效 / 全部（历史含旧内容证据，供回滚）。\n',
         en: 'EVIDENCE column = current / total (history keeps old-content evidence for rollback).\n',

package/dist/cli/lib/record-evolve-outcome.js

@@ -2,26 +2,35 @@ import { resolve } from 'node:path';
 import { createOverlayReportStore } from '../../server/report-store.js';
 import { projectReportsDir, globalReportsDir } from '../../eval-core/measurement-dirs.js';
 import { computeVerdict } from '../../eval-core/verdict.js';
-import { bootstrapDiffCI, DEFAULT_BOOTSTRAP_ALPHA, DEFAULT_BOOTSTRAP_SAMPLES } from '../../eval-core/bootstrap.js';
+import { bootstrapPairedDiffCI, DEFAULT_BOOTSTRAP_ALPHA, DEFAULT_BOOTSTRAP_SAMPLES } from '../../eval-core/bootstrap.js';
 import { resolveManagedDir, managedDir, loadAllManagedRecords, appendManagedEvidence, rebaselineManagedContentHash, buildEvidenceRef, probeSourceState, } from '../../managed/index.js';
 /**
  * round-bestRound vs round-0 的忠实 verdict：复刻 eval 管线（evaluation-reporting.ts）对两变体抽
- * per-sample composite → bootstrapDiffCI → computeVerdict，与 `omk eval --bootstrap` 同口径、同 α / 重采样数。
+ * per-sample composite → 按 sample 配对 bootstrapPairedDiffCI → computeVerdict，与 `omk eval --bootstrap`
+ * 同口径、同 α / 重采样数。
  * 不自造门限：评委是否显著、是否 PROGRESS 全交给既有 computeVerdict。
  */
 function winnerVerdict(report, winnerVariant) {
     const baseline = 'round-0';
-    const scoresOf = (v) => report.results
-        .map((r) => r.variants[v])
-        .filter((e) => !!e && typeof e.compositeScore === 'number' && e.compositeScore > 0)
-        .map((e) => e.compositeScore);
-    const ctrl = scoresOf(baseline);
-    const treat = scoresOf(winnerVariant);
-    const pairComparisons = ctrl.length >= 2 && treat.length >= 2
+    // 按 sample **配对**(与 evaluation-reporting 主 A/B 同口径 —— 本函数职责就是复刻 eval 管线):baseline 与
+    // winner 在同一 sample 上都可测(composite > 0)才入对。同一 sample 两版分数正相关,配对 bootstrap 收紧 diff
+    // CI;独立重采样会高估方差、保守失功效。diff = b − a = winner − baseline。
+    const compositeOf = (v, r) => {
+        const e = r.variants[v];
+        return e && typeof e.compositeScore === 'number' && e.compositeScore > 0 ? e.compositeScore : undefined;
+    };
+    const pairs = [];
+    for (const r of report.results) {
+        const a = compositeOf(baseline, r);
+        const b = compositeOf(winnerVariant, r);
+        if (a !== undefined && b !== undefined)
+            pairs.push({ a, b });
+    }
+    const pairComparisons = pairs.length >= 2
         ? [{
                 control: baseline,
                 treatment: winnerVariant,
-                diffBootstrapCI: bootstrapDiffCI(ctrl, treat, DEFAULT_BOOTSTRAP_ALPHA, DEFAULT_BOOTSTRAP_SAMPLES),
+                diffBootstrapCI: bootstrapPairedDiffCI(pairs, DEFAULT_BOOTSTRAP_ALPHA, DEFAULT_BOOTSTRAP_SAMPLES),
             }]
         : undefined;
     const slice = {

package/dist/eval-core/bootstrap.d.ts

@@ -17,8 +17,14 @@
  *   - bootstrapDiffCI: CI for the difference (B - A); 0 outside the CI = significant
  *   - bootstrapWithMetric: generic interface so saturation analysis can reuse
  *
- * Reproducibility: pass a fixed `seed` to get deterministic CIs across runs.
- * Without a seed we use Math.random() — fine for production but not for tests.
+ * Reproducibility: CIs are **deterministic by default** — when no `seed` is passed,
+ * a fixed `DEFAULT_BOOTSTRAP_SEED` is used, so the same eval run twice yields
+ * byte-identical CIs (and a stable verdict near the significance boundary). This is
+ * a measurement-validity requirement: an unseeded `Math.random()` would let the
+ * `significant` flag flip between identical runs. Library callers (and specific paths
+ * such as `eval gold compare --seed`) may pass an explicit `seed` to vary the draw; the
+ * main `omk eval` deliberately exposes no seed knob — a fixed default also prevents
+ * seed-shopping for significance.
  */
 export interface BootstrapCI {
     /** Lower bound of the CI. */
@@ -42,6 +48,21 @@ export interface BootstrapDiffCI extends BootstrapCI {
 export declare const DEFAULT_BOOTSTRAP_SAMPLES = 1000;
 /** Default significance level; 0.05 → 95% CI. */
 export declare const DEFAULT_BOOTSTRAP_ALPHA = 0.05;
+/**
+ * Fixed default bootstrap seed → CIs are **deterministic by default** (omk default-strict:
+ * reproducibility affects verdict validity, so it is on by default, not opt-in). The specific
+ * value is arbitrary — only that it is **fixed** matters; it is an implementation detail, not a
+ * user-facing constant, so unlike DEFAULT_BOOTSTRAP_SAMPLES / α it is neither cited in docs nor
+ * guarded by `doc-constants-drift.test.ts`. Callers wanting a different draw pass an explicit `seed`.
+ */
+export declare const DEFAULT_BOOTSTRAP_SEED = 20260616;
+/**
+ * Confidence-level label for display: `(1 − α)·100%`. Multiple-comparison (Bonferroni) correction
+ * shrinks a pairwise α to α/K and widens the CI accordingly — the label must track α so a corrected
+ * (wider) interval is never mislabeled "95%". No alpha (single comparison / classic A-B) ⇒ nominal 95%.
+ * Shared single source for the HTML renderer and the `omk eval` CLI verdict so both read one scale.
+ */
+export declare function ciLevelLabel(alpha?: number): string;
 /**
  * Bootstrap confidence interval for the mean of a single sample.
  *
@@ -68,6 +89,36 @@ export declare function bootstrapMeanCI(scores: number[], alpha?: number, sample
  * @returns        BootstrapDiffCI with low/high of (B - A) and significant flag.
  */
 export declare function bootstrapDiffCI(scoresA: number[], scoresB: number[], alpha?: number, samples?: number, seed?: number): BootstrapDiffCI;
+/**
+ * Bootstrap CI for the *difference* of two means on **paired** observations — when A and B
+ * are two measurements of the **same unit** (e.g. control vs treatment on the same sample,
+ * or original vs alternate judge prompt on the same response). Resamples the **pair indices
+ * jointly** and averages each pair's `b - a`, so the within-pair correlation is preserved.
+ *
+ * Why paired (vs `bootstrapDiffCI`'s independent resampling): when A and B move together
+ * across units (the usual case — the same sample scored by two variants is positively
+ * correlated), much of each group's variance is shared and cancels in the per-pair diff.
+ * The independent (unpaired) bootstrap ignores that, over-states the diff's variance, and
+ * widens the CI — *conservative*, costing real power. Use paired whenever the design is
+ * paired; use `bootstrapDiffCI` only for genuinely independent groups (or where a deliberate
+ * conservative bias is wanted). The point estimate is identical (mean of per-pair diffs =
+ * difference of paired means); only the CI tightens.
+ *
+ * `significant` is derived from the **rounded** `low`/`high` (the persisted bounds), so the
+ * flag never contradicts what is stored / displayed: a CI that rounds to include 0 reads as
+ * not-significant. (Computing it on the unrounded bounds would let the JSON say `low: 0,
+ * significant: true` — a self-contradictory `CI=[0, …]` that downstream `computeVerdict` and
+ * external consumers cannot reconcile.) Matches `bootstrapDiffCI`.
+ *
+ * @param pairs    Aligned observations; `a` = control/baseline, `b` = treatment. diff = b - a.
+ * @param alpha    Significance level. Default 0.05.
+ * @param samples  Bootstrap resamples. Default 1000.
+ * @param seed     Optional seed (deterministic by default — see module header).
+ */
+export declare function bootstrapPairedDiffCI(pairs: Array<{
+    a: number;
+    b: number;
+}>, alpha?: number, samples?: number, seed?: number): BootstrapDiffCI;
 /**
  * Generic bootstrap CI for an arbitrary sample-level metric. Used by
  * saturation analysis to get CI on metrics like stddev or

package/dist/eval-core/bootstrap.js

@@ -17,8 +17,14 @@
  *   - bootstrapDiffCI: CI for the difference (B - A); 0 outside the CI = significant
  *   - bootstrapWithMetric: generic interface so saturation analysis can reuse
  *
- * Reproducibility: pass a fixed `seed` to get deterministic CIs across runs.
- * Without a seed we use Math.random() — fine for production but not for tests.
+ * Reproducibility: CIs are **deterministic by default** — when no `seed` is passed,
+ * a fixed `DEFAULT_BOOTSTRAP_SEED` is used, so the same eval run twice yields
+ * byte-identical CIs (and a stable verdict near the significance boundary). This is
+ * a measurement-validity requirement: an unseeded `Math.random()` would let the
+ * `significant` flag flip between identical runs. Library callers (and specific paths
+ * such as `eval gold compare --seed`) may pass an explicit `seed` to vary the draw; the
+ * main `omk eval` deliberately exposes no seed knob — a fixed default also prevents
+ * seed-shopping for significance.
  */
 /**
  * Default number of bootstrap resamples. Every eval path uses this unless
@@ -28,6 +34,24 @@
 export const DEFAULT_BOOTSTRAP_SAMPLES = 1000;
 /** Default significance level; 0.05 → 95% CI. */
 export const DEFAULT_BOOTSTRAP_ALPHA = 0.05;
+/**
+ * Fixed default bootstrap seed → CIs are **deterministic by default** (omk default-strict:
+ * reproducibility affects verdict validity, so it is on by default, not opt-in). The specific
+ * value is arbitrary — only that it is **fixed** matters; it is an implementation detail, not a
+ * user-facing constant, so unlike DEFAULT_BOOTSTRAP_SAMPLES / α it is neither cited in docs nor
+ * guarded by `doc-constants-drift.test.ts`. Callers wanting a different draw pass an explicit `seed`.
+ */
+export const DEFAULT_BOOTSTRAP_SEED = 20260616;
+/**
+ * Confidence-level label for display: `(1 − α)·100%`. Multiple-comparison (Bonferroni) correction
+ * shrinks a pairwise α to α/K and widens the CI accordingly — the label must track α so a corrected
+ * (wider) interval is never mislabeled "95%". No alpha (single comparison / classic A-B) ⇒ nominal 95%.
+ * Shared single source for the HTML renderer and the `omk eval` CLI verdict so both read one scale.
+ */
+export function ciLevelLabel(alpha) {
+    const pct = (1 - (alpha ?? DEFAULT_BOOTSTRAP_ALPHA)) * 100;
+    return `${Number.isInteger(pct) ? pct : Number(pct.toFixed(1))}%`;
+}
 /** Mulberry32 PRNG — seedable, deterministic for tests. */
 function mulberry32(seed) {
     let s = seed >>> 0;
@@ -40,9 +64,9 @@ function mulberry32(seed) {
     };
 }
 function makeRng(seed) {
-    if (seed == null)
-        return Math.random;
-    return mulberry32(seed);
+    // 默认确定性:无显式 seed 时退 DEFAULT_BOOTSTRAP_SEED(而非 Math.random)——否则同一 eval 两跑会得到
+    // 不同 CI,临界点 significant 翻转 → verdict 不可复现。见模块头 Reproducibility。
+    return mulberry32(seed ?? DEFAULT_BOOTSTRAP_SEED);
 }
 /** Sample n indices with replacement from [0, length) using the given PRNG. */
 function resampleIndices(length, n, rng) {
@@ -150,6 +174,59 @@ export function bootstrapDiffCI(scoresA, scoresB, alpha = DEFAULT_BOOTSTRAP_ALPH
         significant: !(low <= 0 && 0 <= high),
     };
 }
+/**
+ * Bootstrap CI for the *difference* of two means on **paired** observations — when A and B
+ * are two measurements of the **same unit** (e.g. control vs treatment on the same sample,
+ * or original vs alternate judge prompt on the same response). Resamples the **pair indices
+ * jointly** and averages each pair's `b - a`, so the within-pair correlation is preserved.
+ *
+ * Why paired (vs `bootstrapDiffCI`'s independent resampling): when A and B move together
+ * across units (the usual case — the same sample scored by two variants is positively
+ * correlated), much of each group's variance is shared and cancels in the per-pair diff.
+ * The independent (unpaired) bootstrap ignores that, over-states the diff's variance, and
+ * widens the CI — *conservative*, costing real power. Use paired whenever the design is
+ * paired; use `bootstrapDiffCI` only for genuinely independent groups (or where a deliberate
+ * conservative bias is wanted). The point estimate is identical (mean of per-pair diffs =
+ * difference of paired means); only the CI tightens.
+ *
+ * `significant` is derived from the **rounded** `low`/`high` (the persisted bounds), so the
+ * flag never contradicts what is stored / displayed: a CI that rounds to include 0 reads as
+ * not-significant. (Computing it on the unrounded bounds would let the JSON say `low: 0,
+ * significant: true` — a self-contradictory `CI=[0, …]` that downstream `computeVerdict` and
+ * external consumers cannot reconcile.) Matches `bootstrapDiffCI`.
+ *
+ * @param pairs    Aligned observations; `a` = control/baseline, `b` = treatment. diff = b - a.
+ * @param alpha    Significance level. Default 0.05.
+ * @param samples  Bootstrap resamples. Default 1000.
+ * @param seed     Optional seed (deterministic by default — see module header).
+ */
+export function bootstrapPairedDiffCI(pairs, alpha = DEFAULT_BOOTSTRAP_ALPHA, samples = DEFAULT_BOOTSTRAP_SAMPLES, seed) {
+    if (pairs.length === 0) {
+        return { low: 0, high: 0, estimate: 0, samples: 0, significant: false };
+    }
+    const n = pairs.length;
+    const diffs = pairs.map((p) => p.b - p.a);
+    const rng = makeRng(seed);
+    const resampleDiffMeans = new Array(samples);
+    for (let s = 0; s < samples; s++) {
+        const idx = resampleIndices(n, n, rng);
+        let sum = 0;
+        for (const i of idx)
+            sum += diffs[i];
+        resampleDiffMeans[s] = sum / n;
+    }
+    resampleDiffMeans.sort((a, b) => a - b);
+    const low = round4(sortedQuantile(resampleDiffMeans, alpha / 2));
+    const high = round4(sortedQuantile(resampleDiffMeans, 1 - alpha / 2));
+    return {
+        low,
+        high,
+        estimate: round4(mean(diffs)),
+        samples,
+        // significant 与持久化的(舍入)边界一致 —— 见函数头:绝不出现「low:0 但 significant:true」自相矛盾。
+        significant: !(low <= 0 && 0 <= high),
+    };
+}
 /**
  * Generic bootstrap CI for an arbitrary sample-level metric. Used by
  * saturation analysis to get CI on metrics like stddev or

package/dist/eval-core/evaluation-reporting.js

@@ -8,7 +8,7 @@ import { indexReportWrite } from './artifact-index.js';
 import { buildVariantSummary } from './schema.js';
 import { buildVariantConfig, resolveExecutionStrategy } from './execution-strategy.js';
 import { getJudgePromptHash } from '../grading/judge.js';
-import { bootstrapMeanCI, bootstrapDiffCI, DEFAULT_BOOTSTRAP_ALPHA, DEFAULT_BOOTSTRAP_SAMPLES, } from './bootstrap.js';
+import { bootstrapMeanCI, bootstrapPairedDiffCI, DEFAULT_BOOTSTRAP_ALPHA, DEFAULT_BOOTSTRAP_SAMPLES, } from './bootstrap.js';
 import { getExecutorRuntimeFingerprint } from '../executors/runtime-fingerprint.js';
 const __dirname = dirname(fileURLToPath(import.meta.url));
 function findPackageJson(startDir) {
@@ -121,6 +121,9 @@ export function aggregateReport({ runId, variants, model, judgeModel, noJudge, e
     const bootstrapSamples = request?.bootstrapSamples ?? DEFAULT_BOOTSTRAP_SAMPLES;
     let pairComparisons;
     if (bootstrapEnabled) {
+        // 不变量(见 grading/layered-scores.ts):composite 在至少一层可测时恒 ≥ 1,`compositeScore === 0`
+        // 当且仅当该样本**无任何可测层**(真·缺测,如纯评委样本且评委失败)。故 `> 0` 过滤精确剔除非测量、
+        // 绝不丢"低分内容"(评委失败已在上游当缺测,不会以 0 进 composite)。下同(control / treatment)。
         for (const variant of variants) {
             const entries = Object.values(results).map((r) => r[variant]).filter(Boolean);
             const compositeScores = entries
@@ -135,23 +138,42 @@ export function aggregateReport({ runId, variants, model, judgeModel, noJudge, e
         if (variants.length >= 2) {
             pairComparisons = [];
             const controlName = variants[0];
-            const controlEntries = Object.values(results).map((r) => r[controlName]).filter(Boolean);
-            const controlScores = controlEntries
-                .filter((e) => typeof e.compositeScore === 'number' && e.compositeScore > 0)
-                .map((e) => e.compositeScore);
+            const sampleRecords = Object.values(results);
+            // **配对** diff CI:A/B 是同一批 sample 分别过 control / treatment(配对设计),按 sample 对齐 ——
+            // 同一 sample 上 control 与 treatment 都可测(composite > 0,见上不变量)才入对。同一 sample 两 variant
+            // 的分数正相关,配对 bootstrap 据此抵消共有方差、收紧 diff CI、更有功效;旧的独立(非配对)重采样高估
+            // 方差、CI 偏宽、保守失功效(见 bootstrapPairedDiffCI)。点估计不变,只收紧 CI。
+            // 先收齐每个 treatment 的配对数据;只有 ≥2 对(否则无 CI 可算)才算一个真正被检验的比较。
+            const eligible = [];
             for (let i = 1; i < variants.length; i++) {
                 const treatmentName = variants[i];
-                const treatmentEntries = Object.values(results).map((r) => r[treatmentName]).filter(Boolean);
-                const treatmentScores = treatmentEntries
-                    .filter((e) => typeof e.compositeScore === 'number' && e.compositeScore > 0)
-                    .map((e) => e.compositeScore);
-                if (controlScores.length >= 2 && treatmentScores.length >= 2) {
-                    pairComparisons.push({
-                        control: controlName,
-                        treatment: treatmentName,
-                        diffBootstrapCI: bootstrapDiffCI(controlScores, treatmentScores, DEFAULT_BOOTSTRAP_ALPHA, bootstrapSamples),
-                    });
+                const pairs = [];
+                for (const r of sampleRecords) {
+                    const c = r[controlName];
+                    const t = r[treatmentName];
+                    const a = c && typeof c.compositeScore === 'number' && c.compositeScore > 0 ? c.compositeScore : undefined;
+                    const b = t && typeof t.compositeScore === 'number' && t.compositeScore > 0 ? t.compositeScore : undefined;
+                    if (a !== undefined && b !== undefined)
+                        pairs.push({ a, b });
                 }
+                if (pairs.length >= 2)
+                    eligible.push({ treatment: treatmentName, pairs });
+            }
+            // 多重比较(Bonferroni)校正:同时检验 K 个 treatment-vs-control 假设时,family-wise 假阳性随 K 膨胀
+            // (computeVerdict 的 worst-case roll-up 取最差 —— 任一对假阳即拉高总判定)。每对 CI 用 α/K(K = 实际
+            // 产出 CI 的比较数)把 family-wise error 压回名义 α。K=1(单 treatment / 经典 A-B)即 α 不变、与历史单对
+            // 口径逐字节一致,此时不写 alpha 字段(渲染按名义 95% CI,既有报告 / 快照不动)。CI 与 significant 同在 α/K
+            // 下算,二者自洽(绝不出现 CI 含 0 却 significant 的矛盾)。注:K 很大时 α/K 落到极端分位,1000 重采样的
+            // 尾部分位偏粗,大 K 慎读 —— 不在本 PR 提采样数。
+            const familySize = eligible.length;
+            const perComparisonAlpha = familySize >= 1 ? DEFAULT_BOOTSTRAP_ALPHA / familySize : DEFAULT_BOOTSTRAP_ALPHA;
+            for (const { treatment, pairs } of eligible) {
+                pairComparisons.push({
+                    control: controlName,
+                    treatment,
+                    diffBootstrapCI: bootstrapPairedDiffCI(pairs, perComparisonAlpha, bootstrapSamples),
+                    ...(familySize >= 2 ? { alpha: perComparisonAlpha } : {}),
+                });
             }
         }
     }

package/dist/eval-core/schema.js

@@ -88,7 +88,8 @@ export function buildVariantResult(execResult, gradeResult, options) {
                 layeredScores.factScore = assertionFact != null
                     ? Number(((assertionFact + hardScore) / 2).toFixed(2))
                     : hardScore;
-                // Recompute composite from updated layers (保留 0 分,仅过滤真正缺失)
+                // Recompute composite from updated layers. 仅过滤 null/undefined(缺测层);评委失败时 judgeScore
+                // 本就是 undefined(见 grading 的 score=0 修复:评委失败=缺测、不以 0 进层),不存在「0 分」要保留。
                 const scores = [layeredScores.factScore, layeredScores.behaviorScore, layeredScores.judgeScore].filter((s) => s != null);
                 compositeScore = scores.length > 0 ? Number((scores.reduce((a, b) => a + b, 0) / scores.length).toFixed(2)) : compositeScore;
             }
@@ -204,8 +205,9 @@ export function buildVariantSummary(entries) {
             };
         })(),
         ...(() => {
-            // 保留 0 分用例(评委打"完全不合格"是合法低分,不是缺失)。
-            // 仅 filter null / undefined(真正缺数据,如该 sample 无对应断言或未配 judge)。
+            // 各层分量为 null/undefined = 该层缺测(无对应断言 / 未配 judge / 评委失败)。评委失败已在 grading 层
+            // 当缺测、judgeScore 留 undefined(见 score=0 修复),不存在「0 分内容」要保留;评委有效分恒 1-5。
+            // filter != null 即精确剔除缺测层。
             const factScores = ok.map((e) => e.layeredScores?.factScore).filter((s) => s != null);
             const behaviorScores = ok.map((e) => e.layeredScores?.behaviorScore).filter((s) => s != null);
             const judgeScores = ok.map((e) => e.layeredScores?.judgeScore).filter((s) => s != null);