oh-my-knowledge 0.40.0 → 0.42.0

package/dist/grading/index.d.ts

@@ -45,10 +45,10 @@ interface GradeOptions {
      */
     judgeRepeat?: number;
     /**
-     * v0.21 length-debias toggle. Defaults to true — judge prompt includes the
-     * "length is not a quality signal" instruction, prompt template version is
-     * v3-cot-length. Set false (via `--no-debias-length`) to revert to the
-     * legacy v2-cot prompt for reproducing historical reports.
+     * length-debias toggle. Defaults to true — judge prompt includes the
+     * "length is not a quality signal" instruction. Set false (via `--no-debias-length`)
+     * to drop it (the debias-off prompt variant) for reproducing older no-length-debias
+     * reports. The presentation/tone neutrality instruction is always on regardless.
      */
     lengthDebias?: boolean;
 }

package/dist/grading/judge.d.ts

@@ -1,13 +1,5 @@
 import type { DimensionResult, ExecutorFn, JudgeAgreement, JudgeConfig, ToolCallInfo, TurnInfo } from '../types/index.js';
-export declare function buildJudgePrompt(prompt: string, rubric: string, output: string, traceSummary: string | null, lengthDebias?: boolean): string;
-/**
- * Stable hash of the judge prompt template. Saved into ReportMeta.judgePromptHash so
- * downstream readers can detect "the judge prompt changed between these two reports".
- *
- * `lengthDebias` defaults to true (v0.21+ default). Pass false when running under
- * `--no-debias-length` so the hash matches historical v2-cot reports.
- */
-export declare function getJudgePromptHash(lengthDebias?: boolean): string;
+export { buildJudgePrompt, getJudgePromptHash } from '../shared/llm-prompts/judge-prompts.js';
 interface LlmJudgeOptions {
     output: string;
     rubric: string;
@@ -17,9 +9,10 @@ interface LlmJudgeOptions {
     traceSummary?: string | null;
     /**
      * When true (default), the judge prompt includes an explicit
-     * "length is not a quality signal" instruction. Pass false to fall back to
-     * the legacy v2-cot prompt — only useful for reproducing pre-v0.21 reports
-     * or running A/B comparisons with alternate length-debias settings.
+     * "length is not a quality signal" instruction. Pass false to drop it (the
+     * debias-off prompt variant) — only useful for reproducing older no-length-debias
+     * reports or running A/B comparisons with alternate length-debias settings.
+     * (The presentation/tone neutrality instruction is always on, independent of this.)
      */
     lengthDebias?: boolean;
 }
@@ -60,7 +53,7 @@ export declare function judgeId(config: JudgeConfig): string;
 export declare function computeJudgeAgreement(judgeScores: number[][]): JudgeAgreement;
 /**
  * Judge a single (output, rubric) pair with N judge models in parallel. Each judge
- * may use a different executor (e.g. claude:opus + openai:gpt-4o + gemini:pro). Each
+ * may use a different executor (e.g. claude:opus + openai-api:gpt-4o + gemini:pro). Each
  * judge can also be repeated `judgeRepeat` times — final per-judge score is its mean.
  *
  * Returns: aggregate DimensionResult (score = mean across judges; this is the "consensus"
@@ -72,4 +65,3 @@ export declare function computeJudgeAgreement(judgeScores: number[][]): JudgeAgr
  * provide the raw ensemble so downstream can recompute.
  */
 export declare function llmJudgeEnsemble(options: LlmJudgeOptions, judges: JudgeConfig[], executorByName: (name: string) => ExecutorFn, judgeRepeat?: number): Promise<DimensionResult>;
-export {};

package/dist/grading/judge.js

@@ -1,4 +1,7 @@
-import { createHash } from 'node:crypto';
+import { buildJudgePrompt, JUDGE_SYSTEM_PROMPT } from '../shared/llm-prompts/judge-prompts.js';
+// 评分类 prompt 已收口到 shared/llm-prompts/judge-prompts.ts(单一来源 + prompt-registry 冻结)。
+// 这里 re-export 保对外 API 不破:既有消费方仍从 grading/judge.js import 这两个符号。
+export { buildJudgePrompt, getJudgePromptHash } from '../shared/llm-prompts/judge-prompts.js';
 function extractFirstJsonObject(text) {
     const start = text.indexOf('{');
     if (start === -1)
@@ -46,92 +49,6 @@ function salvageJudgeResponse(text) {
         reasoning: text.trim().slice(0, 2000),
     };
 }
-/**
- * Judge prompt template version.
- *
- *  - 'v2-cot'         — legacy; CoT scoring without explicit length-debias instruction.
- *                       Kept for `--no-debias-length` so users can reproduce historical
- *                       reports byte-for-byte.
- *  - 'v3-cot-length'  — adds a paragraph telling the judge that length is not a quality
- *                       signal. Default on (research consistently shows
- *                       LLM judges over-weight verbosity; explicit instruction mitigates).
- *
- * Bump when the prompt's intent or structure changes meaningfully — reports tagged
- * with the same hash are score-comparable; mismatched hashes mean "we changed how we
- * ask the judge to think" and should not be compared blind.
- */
-// 版本字符串内嵌"判官能看到什么"的语义。bump 时机:
-//   v2-cot           : 仅看 output + rubric + 工具名分布(早期 buildTraceSummary)
-//   v3-cot-length    : 加了 length-debias 指令(v0.21)
-//   v3-cot-toolargs  : 加了 tool input 预览(本次,BREAKING-COMPARABILITY)
-//   v4-cot-len-args  : v3-cot-length 的同步升级(BREAKING-COMPARABILITY)
-//   bump 原因:之前 trace 只给 tool 名 + 分布,wrapper-style skill(mcporter / code-host CLI 等)
-//   被判官当成"只调了 Bash,没用 skill 指定的 MCP 工具"——结论事实错误。
-//   tool input 预览让判官能识别 `Bash: mcporter --tool skylark_xxx` 内的真实语义调用。
-//
-// ⚠️ 命名约定(下次 bump 时执行):
-//   当前 debias OFF: v3-* / debias ON: v4-* 编号不对称(历史上 debias OFF 跑 v2→v3,
-//   debias ON 跑 v3→v4)。这次再 bump 会让两条线持续偏移 v4/v5、v5/v6、…
-//
-//   下次 bump 时统一为单一主序号 + features 后缀,如:
-//     JUDGE_PROMPT_VERSION_DEBIAS_OFF = 'v5-cot-<features>'
-//     JUDGE_PROMPT_VERSION_DEBIAS_ON  = 'v5-cot-<features>-len'
-//   这样 hash 测试自然分两条,主序号清晰对齐,features 字段独立描述差异。
-//   本 PR 不在意命名,只是把约定写下来,避免下次又跟着错误偏移走。
-const JUDGE_PROMPT_VERSION_DEBIAS_OFF = 'v3-cot-toolargs';
-const JUDGE_PROMPT_VERSION_DEBIAS_ON = 'v4-cot-len-args';
-const JUDGE_SYSTEM_PROMPT = '你是一个严格的 AI 输出质量评审员。先逐条对照评分标准做推理，再给最终分数。只返回 JSON，不要其他内容。';
-const LENGTH_DEBIAS_INSTRUCTION = [
-    '## 重要：长度不是质量信号',
-    '评分时聚焦内容实质与正确性。回答的篇幅、行文丰富度、结构复杂度本身不是质量指标 ——',
-    '简洁正确的回答不应因短而扣分；冗长但偏题或重复的回答不应因长而加分。',
-    '研究显示 LLM 评委容易隐性偏向更长的回答，请在打分前先警觉这一点。',
-].join('\n');
-export function buildJudgePrompt(prompt, rubric, output, traceSummary, lengthDebias = true) {
-    const version = lengthDebias ? JUDGE_PROMPT_VERSION_DEBIAS_ON : JUDGE_PROMPT_VERSION_DEBIAS_OFF;
-    const traceSection = traceSummary
-        ? ['', '## Agent 执行过程', traceSummary, '', '请同时考虑执行过程的合理性（工具选择、步骤效率、错误恢复）。']
-        : [];
-    const debiasSection = lengthDebias ? ['', LENGTH_DEBIAS_INSTRUCTION] : [];
-    return [
-        `请对以下 AI 输出进行质量评分（template ${version}）。`,
-        '',
-        '## 原始任务',
-        prompt,
-        '',
-        '## 评分标准',
-        rubric,
-        '',
-        '## AI 输出',
-        output,
-        ...traceSection,
-        ...debiasSection,
-        '',
-        '## 评分流程',
-        '1. 逐条对照评分标准，先做推理（reasoning）：列出 AI 输出哪些点对应哪条标准，哪些缺失，哪些有歧义。',
-        '2. 基于推理给出最终分数（1-5 的整数）和简短理由。',
-        '',
-        '请返回 JSON（不要包含 markdown 代码块标记）：',
-        '{"reasoning": "<对照标准的逐条推理>", "score": <1-5的整数>, "reason": "<最终结论的简短理由>"}',
-        '',
-        '评分标准：1=完全不达标, 2=部分涉及, 3=基本达标, 4=较好, 5=优秀',
-    ].join('\n');
-}
-/**
- * Stable hash of the judge prompt template. Saved into ReportMeta.judgePromptHash so
- * downstream readers can detect "the judge prompt changed between these two reports".
- *
- * `lengthDebias` defaults to true (v0.21+ default). Pass false when running under
- * `--no-debias-length` so the hash matches historical v2-cot reports.
- */
-export function getJudgePromptHash(lengthDebias = true) {
-    const version = lengthDebias ? JUDGE_PROMPT_VERSION_DEBIAS_ON : JUDGE_PROMPT_VERSION_DEBIAS_OFF;
-    // Hash the template-shaping function source + the version tag together. We hash a
-    // deterministic stringified form of the template (with placeholder inputs) so any
-    // structural edit shows up.
-    const sample = buildJudgePrompt('<P>', '<R>', '<O>', '<T>', lengthDebias);
-    return createHash('sha256').update(version + '\n' + sample).digest('hex').slice(0, 12);
-}
 function getErrorMessage(err) {
     return err instanceof Error ? err.message : String(err);
 }
@@ -412,7 +329,7 @@ export function computeJudgeAgreement(judgeScores) {
 }
 /**
  * Judge a single (output, rubric) pair with N judge models in parallel. Each judge
- * may use a different executor (e.g. claude:opus + openai:gpt-4o + gemini:pro). Each
+ * may use a different executor (e.g. claude:opus + openai-api:gpt-4o + gemini:pro). Each
  * judge can also be repeated `judgeRepeat` times — final per-judge score is its mean.
  *
  * Returns: aggregate DimensionResult (score = mean across judges; this is the "consensus"

package/dist/inputs/eval-config.js

@@ -107,6 +107,9 @@ function validateEvalConfig(parsed, configPath) {
                     throw new Error(`${configPath}: variants[${i}].allowedSkills[${j}] must be a non-empty string`);
                 }
             }
+            if (v.allowedSkills.length > 0) {
+                throw new Error(`${configPath}: variants[${i}].allowedSkills must be [] (strict isolation: no skills) or omitted (no isolation) — a non-empty skill whitelist is no longer supported because it could not be fully isolated (the subagent Skill tool and cwd filesystem channels leaked). For a multi-skill experiment, control the eval environment instead.`);
+            }
             allowedSkills = v.allowedSkills;
         }
         if (seen.has(v.name)) {
@@ -146,11 +149,13 @@ function validateEvalConfig(parsed, configPath) {
     if (obj.judgeModel !== undefined || obj.judgeExecutor !== undefined) {
         throw new Error(`${configPath}: \`judgeModel\` and \`judgeExecutor\` were removed in v0.25 — use \`judgeModels: [{executor, model}]\` instead (single judge is the 1-entry case). See README.`);
     }
+    if (obj.blind !== undefined) {
+        throw new Error(`${configPath}: \`blind\` (judge blind mode) was removed — delete it from your eval.yaml; reports are no longer blinded.`);
+    }
     assertNumberOpt('concurrency');
     assertNumberOpt('timeoutMs');
     assertBoolOpt('noCache');
     assertBoolOpt('noJudge');
-    assertBoolOpt('blind');
     assertStringOpt('mcpConfig');
     assertStringOpt('goldDir');
     assertBoolOpt('bootstrap');
@@ -166,6 +171,11 @@ function validateEvalConfig(parsed, configPath) {
     };
     assertPositiveIntOpt('repeat');
     assertPositiveIntOpt('judgeRepeat');
+    if (obj.holdoutRatio !== undefined) {
+        if (typeof obj.holdoutRatio !== 'number' || !Number.isFinite(obj.holdoutRatio) || obj.holdoutRatio <= 0 || obj.holdoutRatio >= 1) {
+            throw new Error(`${configPath}: holdoutRatio must be a number in (0, 1)`);
+        }
+    }
     if (obj.bootstrapSamples !== undefined) {
         if (typeof obj.bootstrapSamples !== 'number' || !Number.isFinite(obj.bootstrapSamples) || obj.bootstrapSamples < 100) {
             throw new Error(`${configPath}: bootstrapSamples must be a number ≥ 100`);
@@ -247,11 +257,11 @@ function validateEvalConfig(parsed, configPath) {
         timeoutMs: obj.timeoutMs,
         noCache: obj.noCache,
         noJudge: obj.noJudge,
-        blind: obj.blind,
         mcpConfig: obj.mcpConfig,
         variants,
         budget,
         repeat: obj.repeat,
+        holdoutRatio: obj.holdoutRatio,
         judgeRepeat: obj.judgeRepeat,
         bootstrap: obj.bootstrap,
         bootstrapSamples: obj.bootstrapSamples,

package/dist/managed/evidence.js

@@ -13,8 +13,7 @@
  *     `deriveManagedState` 按 contentHash 匹配裁定(重装新内容后旧证据留存供回滚,却不让新内容显得已测)。
  *   - **跨源**:用三级身份消歧把被测 variant 对到受管记录(install 与 eval 命名不一致:记录名是
  *     skill 短名 `review`,而 eval 报告 variant key 可能是整串表达式 `git:HEAD:skills/review`、
- *     eval.yaml 别名 `candidate`、blind 模式的 `A`/`B`)。`applyBlindMode` 盲化 `variants` 但**不**动
- *     `artifactHashes` / `variantConfigs` 的键面,故三级都按真实键工作:
+ *     eval.yaml 别名 `candidate`)。三级都按 `artifactHashes` / `variantConfigs` 的真实键工作:
  *       (a) 显式**同名** variant —— 最强身份(本地 `--treatment <name>` 与 drift);
  *       (b) **结构化源匹配** —— `variantConfigs[].locator(+ref)` 与 `record.source` 对齐(git / 远端 /
  *           别名),即便内容撞哈也能精确消歧;

package/dist/managed/version-scores.js

@@ -33,7 +33,7 @@ export function buildVersionScores(record, reportsById) {
             continue;
         const variant = Object.keys(hashes).find((v) => hashes[v] === ev.contentHash);
         if (!variant)
-            continue; // 匹配不到变体(旧 schema 不绑 / blind 未带哈)→ 跳过
+            continue; // 匹配不到变体(旧 schema 不绑)→ 跳过
         const s = report.summary?.[variant];
         const composite = s?.avgCompositeScore;
         if (typeof composite !== 'number' || Number.isNaN(composite))

package/dist/renderer/html-renderer.js

@@ -459,7 +459,6 @@ export function renderRunDetail(report, lang = DEFAULT_LANG, skillContext) {
     ${m.sampleHashes ? `<span class="meta-tag" title="${t('sampleHashCountDesc', lang)}">${t('sampleHashCount', lang)}: ${Object.keys(m.sampleHashes).length}/${m.sampleCount}</span>` : ''}
     ${m.evaluationFramework ? `<span class="meta-tag" title="${t('evalFrameworkDesc', lang)}">${t('evalFrameworkLabel', lang)}: ${m.evaluationFramework === 'bootstrap' ? t('evalFrameworkBootstrap', lang) : m.evaluationFramework === 'both' ? t('evalFrameworkBoth', lang) : t('evalFrameworkTTest', lang)}</span>` : ''}
     ${renderDebiasModeTag(m.debiasMode, lang)}
-    ${m.blind ? `<span class="meta-tag" style="color:var(--green)" data-i18n="blindLabel">${t('blindLabel', lang)}</span>` : ''}
   </div>`;
     const auditFingerprints = (() => {
         const auditTags = [
@@ -470,13 +469,6 @@ export function renderRunDetail(report, lang = DEFAULT_LANG, skillContext) {
             return '';
         return `<details class="audit-fingerprints"><summary>${lang === 'zh' ? '审计指纹（用于复现校验）' : 'Audit fingerprints (for reproducibility)'}</summary><div class="meta-tags">${auditTags}</div></details>`;
     })();
-    const blindReveal = m.blind ? `
-    <div style="margin:12px 0">
-      <button onclick="document.getElementById('blind-reveal').style.display=document.getElementById('blind-reveal').style.display==='none'?'block':'none'" data-i18n="revealBlind">${t('revealBlind', lang)}</button>
-      <div id="blind-reveal" style="display:none;margin-top:8px;padding:12px;background:var(--bg-surface);border:1px solid var(--border);border-radius:var(--radius)" role="region" aria-label="Blind variant mapping">
-        ${Object.entries(m.blindMap || {}).map(([label, real]) => `<div style="font-size:13px;color:var(--text-secondary)"><strong>Variant ${e(label)}</strong> → ${e(real)}</div>`).join('')}
-      </div>
-    </div>` : '';
     // ──────────── 融合单页:Hero → 结论(verdict+六维) → 折叠次要区 → 统一逐用例 ────────────
     const agentOverview = renderAgentOverview(variants, summary, lang);
     const varianceSection = renderVarianceComparisons(report.variance, lang, Boolean(report.meta.layeredStats), summary);
@@ -503,7 +495,6 @@ export function renderRunDetail(report, lang = DEFAULT_LANG, skillContext) {
         : 'Each sample is both a score card and a functional test: composite + layered scores, assertions, diagnostics, and trace combined';
     const body = `
     ${conclusionPanel}
-    ${blindReveal}
     ${setupFold}
     ${analysisFold}
     <section class="ev-samples">

package/dist/renderer/layout.js

@@ -58,7 +58,7 @@ export const I18N = {
         score: '分数', cost: '执行成本', time: '时间',
         deleteBtnText: '删除', deleteConfirm: '确定删除报告', deleteFail: '删除失败',
         reportTitle: '评测报告', backToList: '← 返回列表',
-        judge: '评委', executor: '执行器', blindLabel: '盲测', revealBlind: '显示变体对应关系',
+        judge: '评委', executor: '执行器',
         dimFact: '📋 事实', dimFactDesc: '输出的事实声明是否正确（规则可验证：关键词匹配、格式校验等断言）',
         dimBehavior: '🛠️ 行为', dimBehaviorDesc: '执行过程是否合规（规则可验证：工具调用路径、轮次限制、成本约束等断言）',
         dimJudge: '💬 LLM 评价', dimJudgeDesc: '请一个 LLM 当评委，读任务执行模型的输出内容，按预先写好的评分规则（英文 rubric）打 1-5 分。主观但能抓到规则断言判不了的"整体好不好"',
@@ -72,7 +72,7 @@ export const I18N = {
         judgeStddev: '评委波动', judgeStddevDesc: '同一份输出让评委评 N 次 (--judge-repeat) 得到 N 个分数的标准差。值低 = 评委对自己很坚定；值高 = 这个分本身就是噪声',
         judgeFailures: '评委失败', judgeFailuresDesc: 'N 次评委评分中返回 score=0（解析失败 / 调用错误）的次数。stddev=0 + failureCount>0 不是"完美一致"，是"大部分炸了"',
         judgeReasoning: '评委推理', judgeReasoningExpand: '展开',
-        ensembleHeader: '多评委评分对比', ensembleDesc: '不同评委模型对同一份输出的独立评分。用于反驳"同模态偏差"',
+        ensembleHeader: '多评委评分对比', ensembleDesc: '不同评委模型对同一份输出的独立评分。跨厂商评委时可反驳"同模态偏差"；单厂商 ensemble 一致性高只是共有偏置，不构成反驳',
         agreementHeader: '跨用例评委一致性', agreementDesc: '在所有评测用例上算的多评委一致性',
         pearsonLabel: '皮尔逊系数 (Pearson)', pearsonDesc: '皮尔逊相关系数：1=完全同向排序，0=无关，-1=完全反向',
         madLabel: '平均绝对差 (MAD)', madDesc: '平均绝对差。1-5 制下 < 0.5 紧密一致, > 1.5 大分歧',
@@ -171,7 +171,7 @@ export const I18N = {
         score: 'Score', cost: 'Execution cost', time: 'Time',
         deleteBtnText: 'Delete', deleteConfirm: 'Delete report', deleteFail: 'Delete failed',
         reportTitle: 'Evaluation Report', backToList: '← Back to list',
-        judge: 'Judge', executor: 'Executor', blindLabel: 'BLIND', revealBlind: 'Reveal variant mapping',
+        judge: 'Judge', executor: 'Executor',
         dimFact: '📋 Fact', dimFactDesc: 'Are factual claims correct (rule-verified: keyword matching, schema checks, etc.)',
         dimBehavior: '🛠️ Behavior', dimBehaviorDesc: 'Is execution compliant (rule-verified: tool paths, turn limits, cost constraints)',
         dimJudge: '💬 LLM judge', dimJudgeDesc: 'A separate LLM acts as judge: reads the task execution model output, scores 1-5 against a predefined rubric. Subjective, catches "overall feel" rule-based assertions miss',
@@ -185,7 +185,7 @@ export const I18N = {
         judgeStddev: 'Judge stddev', judgeStddevDesc: 'Stddev across N judge calls (--judge-repeat). Low = judge is consistent; high = this score itself is noisy',
         judgeFailures: 'Judge failures', judgeFailuresDesc: 'How many of N judge calls returned score=0 (parse / executor failure). stddev=0 + failureCount>0 is NOT "perfect agreement" — it means most calls failed',
         judgeReasoning: 'CoT reasoning', judgeReasoningExpand: 'expand',
-        ensembleHeader: 'Per-judge scores', ensembleDesc: 'Independent scores from different judge models for the same output — refutes same-modality bias',
+        ensembleHeader: 'Per-judge scores', ensembleDesc: 'Independent scores from different judge models for the same output — refutes same-modality bias only when judges span vendors; a single-vendor ensemble agreement is shared bias, not a rebuttal',
         agreementHeader: 'Inter-judge agreement', agreementDesc: 'Cross-sample agreement metrics across all judges in this variant',
         pearsonLabel: 'Pearson', pearsonDesc: 'Pearson correlation: 1=perfect rank agreement, 0=independent, -1=anti-correlated',
         madLabel: 'MAD', madDesc: 'Mean absolute difference. On 1-5 scale: < 0.5 tight, > 1.5 large disagreement',

package/dist/renderer/summary.js

@@ -101,6 +101,30 @@ function computeMedianCVPercent(report) {
     const stab = medianStabilityCV(report);
     return stab ? stab.cv * 100 : null;
 }
+// Verdict caveats（过拟合 / 知识缺口）渲染进 pill —— 让 HTML 报告和 CLI 说同一件事:
+// CLI 在 verbose rationale 里给这两条,HTML 之前只剩一个泛化后的 level、看不到触发原因。
+// 用 result.caveats 的结构化数据 i18n,而不是重解析 zh rationale 串。
+function renderVerdictCaveats(caveats, lang) {
+    if (!caveats)
+        return '';
+    const lines = [];
+    if (caveats.overfitting) {
+        const c = caveats.overfitting;
+        lines.push(lang === 'zh'
+            ? `⚠ 过拟合敞口:${c.variant} 训练 ${c.trainScore.toFixed(2)} / 留出 ${c.holdoutScore.toFixed(2)}(差 ${c.gap.toFixed(2)}),提升可能不泛化`
+            : `⚠ Overfitting: ${c.variant} train ${c.trainScore.toFixed(2)} / holdout ${c.holdoutScore.toFixed(2)} (gap ${c.gap.toFixed(2)}) — gain may not generalize`);
+    }
+    if (caveats.gapSignal) {
+        const g = caveats.gapSignal;
+        const wm = g.testSetHash ? g.testSetHash.slice(0, 8) : (g.testSetPath ?? '');
+        lines.push(lang === 'zh'
+            ? `知识缺口率 ${g.gapRatePct}%(test set ${wm},informational)`
+            : `Knowledge gap ${g.gapRatePct}% (test set ${wm}, informational)`);
+    }
+    if (lines.length === 0)
+        return '';
+    return `<div class="page-verdict-caveats">${lines.map((l) => `<span class="page-verdict-caveat">${e(l)}</span>`).join('')}</div>`;
+}
 export function renderVerdictPill(report, lang) {
     let result;
     try {
@@ -110,8 +134,16 @@ export function renderVerdictPill(report, lang) {
         return '';
     }
     const level = result.level;
-    const pair = result.perPair?.[0];
+    // representative = top-level worst pair（与 CLI 同口径）,不是第一对。多 treatment 报告里
+    // worst pair 不一定是 perPair[0],用它才不会把错的 treatment 名写进结论。fallback 兼容旧路径。
+    const pair = result.representative ?? result.perPair?.[0];
     const oneLine = verdictOneLine(level, lang, pair?.treatment, pair?.control);
+    // Δ/CI 证据必须跟文案指同一对:按 representative 匹配对应的 pairComparison(alpha 也走这对),
+    // 否则多 treatment 报告会出现「文案 t2、数字 t1」的混搭。匹配不到 / 无 representative 时 fallback [0]。
+    const pairComparisons = report.meta?.pairComparisons;
+    const activeComparison = (pair
+        ? pairComparisons?.find((p) => p.treatment === pair.treatment && p.control === pair.control)
+        : undefined) ?? pairComparisons?.[0];
     const tooltip = levelTooltip(level, lang);
     const prefix = lang === 'zh' ? '测评结论' : 'Verdict';
     // 机器可读 enum 永远是 level token; 显示给用户的文字按 lang i18n.
@@ -120,7 +152,7 @@ export function renderVerdictPill(report, lang) {
     // hero 只放「答案」: 分差是 verdict 的核心证据数字, 单独一枚 chip。
     // 评测规模 (用例数 × 轮次) 走「实验配置」section 的 subtitle 那条 canonical 路径,
     // 不在 hero 里重复; CV / CI 走 chip tooltip + 方法学审计 / 波动表。
-    const ci = report.meta?.pairComparisons?.[0]?.diffBootstrapCI;
+    const ci = activeComparison?.diffBootstrapCI;
     const cvPct = computeMedianCVPercent(report);
     const metrics = [];
     if (ci) {
@@ -128,7 +160,7 @@ export function renderVerdictPill(report, lang) {
         const cvSuffix = cvPct != null
             ? (lang === 'zh' ? `;多轮稳定性 CV=${cvPct.toFixed(1)}% (${cvPct < 5 ? '稳' : cvPct <= STABILITY_UNSTABLE_CV * 100 ? '中' : '不稳'})` : `; CV=${cvPct.toFixed(1)}% (${cvPct < 5 ? 'stable' : cvPct <= STABILITY_UNSTABLE_CV * 100 ? 'moderate' : 'unstable'})`)
             : '';
-        const pctLabel = ciLevelLabel(report.meta?.pairComparisons?.[0]?.alpha);
+        const pctLabel = ciLevelLabel(activeComparison?.alpha);
         const ciTipBase = lang === 'zh'
             ? `实验组与对照组综合分均值差(Δ)。bootstrap ${pctLabel} 可信区间 [${ci.low}, ${ci.high}]，${ci.significant ? '不含 0 = 差异显著' : '跨过 0 = 差异不显著'}${cvSuffix}`
             : `Treatment minus control mean composite score (Δ). Bootstrap ${pctLabel} CI [${ci.low}, ${ci.high}], ${ci.significant ? 'excludes 0 ⇒ significant' : 'spans 0 ⇒ not significant'}${cvSuffix}`;
@@ -146,6 +178,7 @@ export function renderVerdictPill(report, lang) {
       <span class="page-verdict-badge"><span class="page-verdict-badge-dot" aria-hidden="true">●</span>${e(levelDisplay)}</span>
       <span class="page-verdict-text">${e(oneLine)}</span>
     </div>
+    ${renderVerdictCaveats(result.caveats, lang)}
     ${metricChips ? `<div class="page-verdict-metrics">${metricChips}</div>` : ''}
   </section>`;
 }
@@ -252,7 +285,7 @@ export function renderHumanAgreement(agreement, lang) {
           <td><strong>Krippendorff α</strong></td>
           <td style="text-align:center;color:${alphaColor}"><strong>${fmt(a.alpha)}</strong></td>
           <td style="text-align:center;font-size:11px">[${fmt(a.alphaCI.low)}, ${fmt(a.alphaCI.high)}]</td>
-          <td style="font-size:12px;color:var(--text-secondary)">${lang === 'zh' ? '主指标，序数加权' : 'primary, ordinal-weighted'}</td>
+          <td style="font-size:12px;color:var(--text-secondary)">${lang === 'zh' ? '主指标，区间加权' : 'primary, interval-weighted'}</td>
         </tr>
         <tr>
           <td>${lang === 'zh' ? '加权 κ' : 'weighted κ'}</td>
@@ -1496,6 +1529,20 @@ function localizedInsightMessage(insight, report, lang) {
                 ? `${count} 个评测用例成本显著高于平均值`
                 : `${count} samples cost materially more than average`;
         }
+        case 'judge_self_preference': {
+            const outv = asArray(details.outputVendors).map(String).join('/') || '同厂商';
+            const calibrated = details.goldCalibrated === true;
+            return lang === 'zh'
+                ? `评委与被测输出同厂商（${outv}）${calibrated ? '（已 gold 校准）' : ''}，存在自我偏好敞口：评委可能给同家族输出打高分`
+                : `Judge is the same vendor as the executor that produced the outputs (${outv})${calibrated ? ' (gold-calibrated)' : ''}; self-preference exposure — the judge may over-score same-family output`;
+        }
+        case 'single_vendor_ensemble': {
+            const n = asArray(details.judgeVendors).length;
+            const calibrated = details.goldCalibrated === true;
+            return lang === 'zh'
+                ? `${n} 个评委同属一个厂商${calibrated ? '（已 gold 校准）' : ''}，ensemble 一致性高只反映共有偏置，不构成对同模型偏置的反驳`
+                : `All ${n} judges are from one vendor${calibrated ? ' (gold-calibrated)' : ''}; high ensemble agreement reflects shared bias, not a rebuttal to same-model bias`;
+        }
         default:
             return lang === 'zh'
                 ? `结构化诊断：${type}`
@@ -1544,6 +1591,14 @@ function localizedSuggestion(insight, lang) {
             return lang === 'zh'
                 ? '重写 agent 断言时优先约束工具路径、关键文件读取和 turns 上限'
                 : 'When rewriting agent assertions, prioritize tool path, key file reads, and turn-limit constraints';
+        case 'judge_self_preference':
+            return lang === 'zh'
+                ? '换跨厂商评委（如 --judge-models openai-api:gpt-4o）消除自我偏好，或挂人工金标校准（omk eval gold compare）量化评委是否可信。注：固定模型的 A/B 差值受影响较小，绝对分 / 跨版本曲线受影响更大'
+                : 'Use a cross-vendor judge (e.g. --judge-models openai-api:gpt-4o) to remove self-preference, or calibrate against human gold (omk eval gold compare). Note: the A/B delta is less affected than absolute scores / cross-version curves';
+        case 'single_vendor_ensemble':
+            return lang === 'zh'
+                ? '把 ensemble 配成跨厂商（如 --judge-models claude:opus,openai-api:gpt-4o），让 agreement 真能反驳同模型偏置'
+                : 'Make the ensemble cross-vendor (e.g. --judge-models claude:opus,openai-api:gpt-4o) so agreement actually rebuts same-model bias';
         default:
             return lang === 'zh'
                 ? '查看结构化 details 字段定位原因'

package/dist/shared/llm-prompts/debias-instructions.d.ts

@@ -0,0 +1,4 @@
+export declare const LENGTH_DEBIAS_INSTRUCTION: string;
+export declare const PRESENTATION_NEUTRALITY_INSTRUCTION: string;
+export declare const RAG_LENGTH_DEBIAS: string;
+export declare const RAG_PRESENTATION_NEUTRALITY: string;

package/dist/shared/llm-prompts/debias-instructions.js

@@ -0,0 +1,44 @@
+// 评委 prompt 的共享去偏 / 中性化指令 —— 单一来源。
+//
+// 此前这几段在 grading/judge.ts(rubric 评委)与 grading/assertions.ts(RAG 评委)各写一份,
+// 改一处漏一处(J3 即如此)。统一收口于此,两侧 import。
+//
+// 两套变体是有意的、不要强并成一条:
+//   - 全版(LENGTH_DEBIAS_INSTRUCTION / PRESENTATION_NEUTRALITY_INSTRUCTION):给 rubric 评委,
+//     带研究依据句;rubric prompt 较短,容得下。被 getJudgePromptHash 冻结,改字节 = BREAKING-COMPARABILITY。
+//   - 短版(RAG_LENGTH_DEBIAS / RAG_PRESENTATION_NEUTRALITY):给 RAG 评委,RAG prompt 本就长,精简表述。
+// 改任一段先想另一段是否同步;改全版还要走冻结 hash 的 bump 流程。
+export const LENGTH_DEBIAS_INSTRUCTION = [
+    '## 重要：长度不是质量信号',
+    '评分时聚焦内容实质与正确性。回答的篇幅、行文丰富度、结构复杂度本身不是质量指标 ——',
+    '简洁正确的回答不应因短而扣分；冗长但偏题或重复的回答不应因长而加分。',
+    '研究显示 LLM 评委容易隐性偏向更长的回答，请在打分前先警觉这一点。',
+].join('\n');
+// 始终开启(不给开关):排版与语气都不是质量信号。措辞严格对称 —— 既不奖励精致 / 自信,也不
+// 因朴素 / 含糊而扣分,避免"抗偏置"本身过度矫正成反向偏置。研究表明 LLM 评委除长度外,还隐性
+// 偏向排版精致(format / markdown bias)与语气自信、自我表扬的回答(谄媚 / 权威偏置)。
+export const PRESENTATION_NEUTRALITY_INSTRUCTION = [
+    '## 重要：排版与语气不是质量信号',
+    '评分只看内容是否对照评分标准、是否正确。Markdown 排版、标题、列表、加粗、表格等呈现形式',
+    '本身不是质量指标 —— 朴素但正确的回答不应因没有排版而扣分；排版精致但偏题或错误的回答不应',
+    '因好看而加分。',
+    '同样，回答的语气、自信程度、是否自我表扬（如「这是最优方案」）也不是质量信号 —— 不要被笃定',
+    '的口吻或自我评价带跑：自信但错误的回答不应高于含糊但正确的回答，一切结论都要回到评分标准',
+    '逐条核实。',
+    '研究显示 LLM 评委容易隐性偏向排版精致、语气自信的回答，请在打分前先警觉这两点。',
+].join('\n');
+// RAG 指标(faithfulness / answer_relevancy / context_recall)的去偏**恒开**,
+// 按 default-strict 不提供关闭开关:`--no-debias-length` 只作用于 rubric 评委 prompt 的长度去偏,
+// 不下探到 RAG —— RAG 评的是内容保真 / 切题 / 召回,放任长度 / 排版 / 语气偏置只会污染这些指标,
+// 没有「关掉它」的正当用例。故 runRagJudge 不收 lengthDebias 参数,两段恒注入。
+export const RAG_LENGTH_DEBIAS = [
+    '## 重要：长度不是质量信号',
+    '评分时聚焦内容实质，不要因输出更长就给更高分。',
+    '简洁正确的回答与冗长正确的回答应得相同分数。',
+].join('\n');
+// 排版 / 语气中性化(format / sycophancy bias),与 RAG_LENGTH_DEBIAS 同 footprint 恒开。
+export const RAG_PRESENTATION_NEUTRALITY = [
+    '## 重要：排版与语气不是质量信号',
+    '评分只看内容是否符合上述标准。排版是否精致、语气是否自信、有无自我表扬都不是质量信号 ——',
+    '不要被笃定的口吻带跑：自信但错误的回答不应高于含糊但正确的回答。',
+].join('\n');

package/dist/shared/llm-prompts/judge-prompts.d.ts

@@ -0,0 +1,30 @@
+export declare const JUDGE_SYSTEM_PROMPT = "\u4F60\u662F\u4E00\u4E2A\u4E25\u683C\u7684 AI \u8F93\u51FA\u8D28\u91CF\u8BC4\u5BA1\u5458\u3002\u5148\u9010\u6761\u5BF9\u7167\u8BC4\u5206\u6807\u51C6\u505A\u63A8\u7406\uFF0C\u518D\u7ED9\u6700\u7EC8\u5206\u6570\u3002\u53EA\u8FD4\u56DE JSON\uFF0C\u4E0D\u8981\u5176\u4ED6\u5185\u5BB9\u3002";
+export declare function buildJudgePrompt(prompt: string, rubric: string, output: string, traceSummary: string | null, lengthDebias?: boolean): string;
+/**
+ * Stable hash of the judge prompt template. Saved into ReportMeta.judgePromptHash so
+ * downstream readers can detect "the judge prompt changed between these two reports".
+ *
+ * `lengthDebias` defaults to true. Pass false (via `--no-debias-length`) to drop the
+ * length-debias instruction; that produces the debias-off prompt variant, whose hash
+ * differs from the default so the two are never compared blind.
+ */
+export declare function getJudgePromptHash(lengthDebias?: boolean): string;
+export declare const SEMANTIC_SIMILARITY_SYSTEM = "\u4F60\u662F\u8BED\u4E49\u76F8\u4F3C\u5EA6\u8BC4\u5BA1\u5458\u3002\u53EA\u8FD4\u56DE JSON\uFF0C\u4E0D\u8981\u5176\u4ED6\u5185\u5BB9\u3002";
+export declare function buildSemanticSimilarityPrompt(reference: string, output: string): string;
+export type RagJudgeType = 'faithfulness' | 'answer_relevancy' | 'context_recall';
+export interface RagJudgeFields {
+    output: string;
+    /** faithfulness: 参考 context。 */
+    context?: string;
+    /** answer_relevancy: 用户问题(sample.prompt)。 */
+    question?: string;
+    /** context_recall: 参考 gold。 */
+    reference?: string;
+}
+/** RAG 评委 system + 用户 prompt。缺字段的早退 / 解析逻辑留在 assertions.ts。 */
+export declare function buildRagJudgePrompt(type: RagJudgeType, fields: RagJudgeFields): {
+    system: string;
+    prompt: string;
+};
+export declare function getSemanticPromptHash(): string;
+export declare function getRagJudgePromptHash(type: RagJudgeType): string;