oh-my-knowledge 0.40.0 → 0.41.0

package/dist/grading/judge.js

@@ -1,4 +1,7 @@
-import { createHash } from 'node:crypto';
+import { buildJudgePrompt, JUDGE_SYSTEM_PROMPT } from '../shared/llm-prompts/judge-prompts.js';
+// 评分类 prompt 已收口到 shared/llm-prompts/judge-prompts.ts(单一来源 + prompt-registry 冻结)。
+// 这里 re-export 保对外 API 不破:既有消费方仍从 grading/judge.js import 这两个符号。
+export { buildJudgePrompt, getJudgePromptHash } from '../shared/llm-prompts/judge-prompts.js';
 function extractFirstJsonObject(text) {
     const start = text.indexOf('{');
     if (start === -1)
@@ -46,92 +49,6 @@ function salvageJudgeResponse(text) {
         reasoning: text.trim().slice(0, 2000),
     };
 }
-/**
- * Judge prompt template version.
- *
- *  - 'v2-cot'         — legacy; CoT scoring without explicit length-debias instruction.
- *                       Kept for `--no-debias-length` so users can reproduce historical
- *                       reports byte-for-byte.
- *  - 'v3-cot-length'  — adds a paragraph telling the judge that length is not a quality
- *                       signal. Default on (research consistently shows
- *                       LLM judges over-weight verbosity; explicit instruction mitigates).
- *
- * Bump when the prompt's intent or structure changes meaningfully — reports tagged
- * with the same hash are score-comparable; mismatched hashes mean "we changed how we
- * ask the judge to think" and should not be compared blind.
- */
-// 版本字符串内嵌"判官能看到什么"的语义。bump 时机:
-//   v2-cot           : 仅看 output + rubric + 工具名分布(早期 buildTraceSummary)
-//   v3-cot-length    : 加了 length-debias 指令(v0.21)
-//   v3-cot-toolargs  : 加了 tool input 预览(本次,BREAKING-COMPARABILITY)
-//   v4-cot-len-args  : v3-cot-length 的同步升级(BREAKING-COMPARABILITY)
-//   bump 原因:之前 trace 只给 tool 名 + 分布,wrapper-style skill(mcporter / code-host CLI 等)
-//   被判官当成"只调了 Bash,没用 skill 指定的 MCP 工具"——结论事实错误。
-//   tool input 预览让判官能识别 `Bash: mcporter --tool skylark_xxx` 内的真实语义调用。
-//
-// ⚠️ 命名约定(下次 bump 时执行):
-//   当前 debias OFF: v3-* / debias ON: v4-* 编号不对称(历史上 debias OFF 跑 v2→v3,
-//   debias ON 跑 v3→v4)。这次再 bump 会让两条线持续偏移 v4/v5、v5/v6、…
-//
-//   下次 bump 时统一为单一主序号 + features 后缀,如:
-//     JUDGE_PROMPT_VERSION_DEBIAS_OFF = 'v5-cot-<features>'
-//     JUDGE_PROMPT_VERSION_DEBIAS_ON  = 'v5-cot-<features>-len'
-//   这样 hash 测试自然分两条,主序号清晰对齐,features 字段独立描述差异。
-//   本 PR 不在意命名,只是把约定写下来,避免下次又跟着错误偏移走。
-const JUDGE_PROMPT_VERSION_DEBIAS_OFF = 'v3-cot-toolargs';
-const JUDGE_PROMPT_VERSION_DEBIAS_ON = 'v4-cot-len-args';
-const JUDGE_SYSTEM_PROMPT = '你是一个严格的 AI 输出质量评审员。先逐条对照评分标准做推理，再给最终分数。只返回 JSON，不要其他内容。';
-const LENGTH_DEBIAS_INSTRUCTION = [
-    '## 重要：长度不是质量信号',
-    '评分时聚焦内容实质与正确性。回答的篇幅、行文丰富度、结构复杂度本身不是质量指标 ——',
-    '简洁正确的回答不应因短而扣分；冗长但偏题或重复的回答不应因长而加分。',
-    '研究显示 LLM 评委容易隐性偏向更长的回答，请在打分前先警觉这一点。',
-].join('\n');
-export function buildJudgePrompt(prompt, rubric, output, traceSummary, lengthDebias = true) {
-    const version = lengthDebias ? JUDGE_PROMPT_VERSION_DEBIAS_ON : JUDGE_PROMPT_VERSION_DEBIAS_OFF;
-    const traceSection = traceSummary
-        ? ['', '## Agent 执行过程', traceSummary, '', '请同时考虑执行过程的合理性（工具选择、步骤效率、错误恢复）。']
-        : [];
-    const debiasSection = lengthDebias ? ['', LENGTH_DEBIAS_INSTRUCTION] : [];
-    return [
-        `请对以下 AI 输出进行质量评分（template ${version}）。`,
-        '',
-        '## 原始任务',
-        prompt,
-        '',
-        '## 评分标准',
-        rubric,
-        '',
-        '## AI 输出',
-        output,
-        ...traceSection,
-        ...debiasSection,
-        '',
-        '## 评分流程',
-        '1. 逐条对照评分标准，先做推理（reasoning）：列出 AI 输出哪些点对应哪条标准，哪些缺失，哪些有歧义。',
-        '2. 基于推理给出最终分数（1-5 的整数）和简短理由。',
-        '',
-        '请返回 JSON（不要包含 markdown 代码块标记）：',
-        '{"reasoning": "<对照标准的逐条推理>", "score": <1-5的整数>, "reason": "<最终结论的简短理由>"}',
-        '',
-        '评分标准：1=完全不达标, 2=部分涉及, 3=基本达标, 4=较好, 5=优秀',
-    ].join('\n');
-}
-/**
- * Stable hash of the judge prompt template. Saved into ReportMeta.judgePromptHash so
- * downstream readers can detect "the judge prompt changed between these two reports".
- *
- * `lengthDebias` defaults to true (v0.21+ default). Pass false when running under
- * `--no-debias-length` so the hash matches historical v2-cot reports.
- */
-export function getJudgePromptHash(lengthDebias = true) {
-    const version = lengthDebias ? JUDGE_PROMPT_VERSION_DEBIAS_ON : JUDGE_PROMPT_VERSION_DEBIAS_OFF;
-    // Hash the template-shaping function source + the version tag together. We hash a
-    // deterministic stringified form of the template (with placeholder inputs) so any
-    // structural edit shows up.
-    const sample = buildJudgePrompt('<P>', '<R>', '<O>', '<T>', lengthDebias);
-    return createHash('sha256').update(version + '\n' + sample).digest('hex').slice(0, 12);
-}
 function getErrorMessage(err) {
     return err instanceof Error ? err.message : String(err);
 }
@@ -412,7 +329,7 @@ export function computeJudgeAgreement(judgeScores) {
 }
 /**
  * Judge a single (output, rubric) pair with N judge models in parallel. Each judge
- * may use a different executor (e.g. claude:opus + openai:gpt-4o + gemini:pro). Each
+ * may use a different executor (e.g. claude:opus + openai-api:gpt-4o + gemini:pro). Each
  * judge can also be repeated `judgeRepeat` times — final per-judge score is its mean.
  *
  * Returns: aggregate DimensionResult (score = mean across judges; this is the "consensus"

package/dist/inputs/eval-config.js

@@ -107,6 +107,9 @@ function validateEvalConfig(parsed, configPath) {
                     throw new Error(`${configPath}: variants[${i}].allowedSkills[${j}] must be a non-empty string`);
                 }
             }
+            if (v.allowedSkills.length > 0) {
+                throw new Error(`${configPath}: variants[${i}].allowedSkills must be [] (strict isolation: no skills) or omitted (no isolation) — a non-empty skill whitelist is no longer supported because it could not be fully isolated (the subagent Skill tool and cwd filesystem channels leaked). For a multi-skill experiment, control the eval environment instead.`);
+            }
             allowedSkills = v.allowedSkills;
         }
         if (seen.has(v.name)) {
@@ -146,11 +149,13 @@ function validateEvalConfig(parsed, configPath) {
     if (obj.judgeModel !== undefined || obj.judgeExecutor !== undefined) {
         throw new Error(`${configPath}: \`judgeModel\` and \`judgeExecutor\` were removed in v0.25 — use \`judgeModels: [{executor, model}]\` instead (single judge is the 1-entry case). See README.`);
     }
+    if (obj.blind !== undefined) {
+        throw new Error(`${configPath}: \`blind\` (judge blind mode) was removed — delete it from your eval.yaml; reports are no longer blinded.`);
+    }
     assertNumberOpt('concurrency');
     assertNumberOpt('timeoutMs');
     assertBoolOpt('noCache');
     assertBoolOpt('noJudge');
-    assertBoolOpt('blind');
     assertStringOpt('mcpConfig');
     assertStringOpt('goldDir');
     assertBoolOpt('bootstrap');
@@ -247,7 +252,6 @@ function validateEvalConfig(parsed, configPath) {
         timeoutMs: obj.timeoutMs,
         noCache: obj.noCache,
         noJudge: obj.noJudge,
-        blind: obj.blind,
         mcpConfig: obj.mcpConfig,
         variants,
         budget,

package/dist/managed/evidence.js

@@ -13,8 +13,7 @@
  *     `deriveManagedState` 按 contentHash 匹配裁定(重装新内容后旧证据留存供回滚,却不让新内容显得已测)。
  *   - **跨源**:用三级身份消歧把被测 variant 对到受管记录(install 与 eval 命名不一致:记录名是
  *     skill 短名 `review`,而 eval 报告 variant key 可能是整串表达式 `git:HEAD:skills/review`、
- *     eval.yaml 别名 `candidate`、blind 模式的 `A`/`B`)。`applyBlindMode` 盲化 `variants` 但**不**动
- *     `artifactHashes` / `variantConfigs` 的键面,故三级都按真实键工作:
+ *     eval.yaml 别名 `candidate`)。三级都按 `artifactHashes` / `variantConfigs` 的真实键工作:
  *       (a) 显式**同名** variant —— 最强身份(本地 `--treatment <name>` 与 drift);
  *       (b) **结构化源匹配** —— `variantConfigs[].locator(+ref)` 与 `record.source` 对齐(git / 远端 /
  *           别名),即便内容撞哈也能精确消歧;

package/dist/managed/version-scores.js

@@ -33,7 +33,7 @@ export function buildVersionScores(record, reportsById) {
             continue;
         const variant = Object.keys(hashes).find((v) => hashes[v] === ev.contentHash);
         if (!variant)
-            continue; // 匹配不到变体(旧 schema 不绑 / blind 未带哈)→ 跳过
+            continue; // 匹配不到变体(旧 schema 不绑)→ 跳过
         const s = report.summary?.[variant];
         const composite = s?.avgCompositeScore;
         if (typeof composite !== 'number' || Number.isNaN(composite))

package/dist/renderer/html-renderer.js

@@ -459,7 +459,6 @@ export function renderRunDetail(report, lang = DEFAULT_LANG, skillContext) {
     ${m.sampleHashes ? `<span class="meta-tag" title="${t('sampleHashCountDesc', lang)}">${t('sampleHashCount', lang)}: ${Object.keys(m.sampleHashes).length}/${m.sampleCount}</span>` : ''}
     ${m.evaluationFramework ? `<span class="meta-tag" title="${t('evalFrameworkDesc', lang)}">${t('evalFrameworkLabel', lang)}: ${m.evaluationFramework === 'bootstrap' ? t('evalFrameworkBootstrap', lang) : m.evaluationFramework === 'both' ? t('evalFrameworkBoth', lang) : t('evalFrameworkTTest', lang)}</span>` : ''}
     ${renderDebiasModeTag(m.debiasMode, lang)}
-    ${m.blind ? `<span class="meta-tag" style="color:var(--green)" data-i18n="blindLabel">${t('blindLabel', lang)}</span>` : ''}
   </div>`;
     const auditFingerprints = (() => {
         const auditTags = [
@@ -470,13 +469,6 @@ export function renderRunDetail(report, lang = DEFAULT_LANG, skillContext) {
             return '';
         return `<details class="audit-fingerprints"><summary>${lang === 'zh' ? '审计指纹（用于复现校验）' : 'Audit fingerprints (for reproducibility)'}</summary><div class="meta-tags">${auditTags}</div></details>`;
     })();
-    const blindReveal = m.blind ? `
-    <div style="margin:12px 0">
-      <button onclick="document.getElementById('blind-reveal').style.display=document.getElementById('blind-reveal').style.display==='none'?'block':'none'" data-i18n="revealBlind">${t('revealBlind', lang)}</button>
-      <div id="blind-reveal" style="display:none;margin-top:8px;padding:12px;background:var(--bg-surface);border:1px solid var(--border);border-radius:var(--radius)" role="region" aria-label="Blind variant mapping">
-        ${Object.entries(m.blindMap || {}).map(([label, real]) => `<div style="font-size:13px;color:var(--text-secondary)"><strong>Variant ${e(label)}</strong> → ${e(real)}</div>`).join('')}
-      </div>
-    </div>` : '';
     // ──────────── 融合单页:Hero → 结论(verdict+六维) → 折叠次要区 → 统一逐用例 ────────────
     const agentOverview = renderAgentOverview(variants, summary, lang);
     const varianceSection = renderVarianceComparisons(report.variance, lang, Boolean(report.meta.layeredStats), summary);
@@ -503,7 +495,6 @@ export function renderRunDetail(report, lang = DEFAULT_LANG, skillContext) {
         : 'Each sample is both a score card and a functional test: composite + layered scores, assertions, diagnostics, and trace combined';
     const body = `
     ${conclusionPanel}
-    ${blindReveal}
     ${setupFold}
     ${analysisFold}
     <section class="ev-samples">

package/dist/renderer/layout.js

@@ -58,7 +58,7 @@ export const I18N = {
         score: '分数', cost: '执行成本', time: '时间',
         deleteBtnText: '删除', deleteConfirm: '确定删除报告', deleteFail: '删除失败',
         reportTitle: '评测报告', backToList: '← 返回列表',
-        judge: '评委', executor: '执行器', blindLabel: '盲测', revealBlind: '显示变体对应关系',
+        judge: '评委', executor: '执行器',
         dimFact: '📋 事实', dimFactDesc: '输出的事实声明是否正确（规则可验证：关键词匹配、格式校验等断言）',
         dimBehavior: '🛠️ 行为', dimBehaviorDesc: '执行过程是否合规（规则可验证：工具调用路径、轮次限制、成本约束等断言）',
         dimJudge: '💬 LLM 评价', dimJudgeDesc: '请一个 LLM 当评委，读任务执行模型的输出内容，按预先写好的评分规则（英文 rubric）打 1-5 分。主观但能抓到规则断言判不了的"整体好不好"',
@@ -72,7 +72,7 @@ export const I18N = {
         judgeStddev: '评委波动', judgeStddevDesc: '同一份输出让评委评 N 次 (--judge-repeat) 得到 N 个分数的标准差。值低 = 评委对自己很坚定；值高 = 这个分本身就是噪声',
         judgeFailures: '评委失败', judgeFailuresDesc: 'N 次评委评分中返回 score=0（解析失败 / 调用错误）的次数。stddev=0 + failureCount>0 不是"完美一致"，是"大部分炸了"',
         judgeReasoning: '评委推理', judgeReasoningExpand: '展开',
-        ensembleHeader: '多评委评分对比', ensembleDesc: '不同评委模型对同一份输出的独立评分。用于反驳"同模态偏差"',
+        ensembleHeader: '多评委评分对比', ensembleDesc: '不同评委模型对同一份输出的独立评分。跨厂商评委时可反驳"同模态偏差"；单厂商 ensemble 一致性高只是共有偏置，不构成反驳',
         agreementHeader: '跨用例评委一致性', agreementDesc: '在所有评测用例上算的多评委一致性',
         pearsonLabel: '皮尔逊系数 (Pearson)', pearsonDesc: '皮尔逊相关系数：1=完全同向排序，0=无关，-1=完全反向',
         madLabel: '平均绝对差 (MAD)', madDesc: '平均绝对差。1-5 制下 < 0.5 紧密一致, > 1.5 大分歧',
@@ -171,7 +171,7 @@ export const I18N = {
         score: 'Score', cost: 'Execution cost', time: 'Time',
         deleteBtnText: 'Delete', deleteConfirm: 'Delete report', deleteFail: 'Delete failed',
         reportTitle: 'Evaluation Report', backToList: '← Back to list',
-        judge: 'Judge', executor: 'Executor', blindLabel: 'BLIND', revealBlind: 'Reveal variant mapping',
+        judge: 'Judge', executor: 'Executor',
         dimFact: '📋 Fact', dimFactDesc: 'Are factual claims correct (rule-verified: keyword matching, schema checks, etc.)',
         dimBehavior: '🛠️ Behavior', dimBehaviorDesc: 'Is execution compliant (rule-verified: tool paths, turn limits, cost constraints)',
         dimJudge: '💬 LLM judge', dimJudgeDesc: 'A separate LLM acts as judge: reads the task execution model output, scores 1-5 against a predefined rubric. Subjective, catches "overall feel" rule-based assertions miss',
@@ -185,7 +185,7 @@ export const I18N = {
         judgeStddev: 'Judge stddev', judgeStddevDesc: 'Stddev across N judge calls (--judge-repeat). Low = judge is consistent; high = this score itself is noisy',
         judgeFailures: 'Judge failures', judgeFailuresDesc: 'How many of N judge calls returned score=0 (parse / executor failure). stddev=0 + failureCount>0 is NOT "perfect agreement" — it means most calls failed',
         judgeReasoning: 'CoT reasoning', judgeReasoningExpand: 'expand',
-        ensembleHeader: 'Per-judge scores', ensembleDesc: 'Independent scores from different judge models for the same output — refutes same-modality bias',
+        ensembleHeader: 'Per-judge scores', ensembleDesc: 'Independent scores from different judge models for the same output — refutes same-modality bias only when judges span vendors; a single-vendor ensemble agreement is shared bias, not a rebuttal',
         agreementHeader: 'Inter-judge agreement', agreementDesc: 'Cross-sample agreement metrics across all judges in this variant',
         pearsonLabel: 'Pearson', pearsonDesc: 'Pearson correlation: 1=perfect rank agreement, 0=independent, -1=anti-correlated',
         madLabel: 'MAD', madDesc: 'Mean absolute difference. On 1-5 scale: < 0.5 tight, > 1.5 large disagreement',

package/dist/renderer/summary.js

@@ -252,7 +252,7 @@ export function renderHumanAgreement(agreement, lang) {
           <td><strong>Krippendorff α</strong></td>
           <td style="text-align:center;color:${alphaColor}"><strong>${fmt(a.alpha)}</strong></td>
           <td style="text-align:center;font-size:11px">[${fmt(a.alphaCI.low)}, ${fmt(a.alphaCI.high)}]</td>
-          <td style="font-size:12px;color:var(--text-secondary)">${lang === 'zh' ? '主指标，序数加权' : 'primary, ordinal-weighted'}</td>
+          <td style="font-size:12px;color:var(--text-secondary)">${lang === 'zh' ? '主指标，区间加权' : 'primary, interval-weighted'}</td>
         </tr>
         <tr>
           <td>${lang === 'zh' ? '加权 κ' : 'weighted κ'}</td>
@@ -1496,6 +1496,20 @@ function localizedInsightMessage(insight, report, lang) {
                 ? `${count} 个评测用例成本显著高于平均值`
                 : `${count} samples cost materially more than average`;
         }
+        case 'judge_self_preference': {
+            const outv = asArray(details.outputVendors).map(String).join('/') || '同厂商';
+            const calibrated = details.goldCalibrated === true;
+            return lang === 'zh'
+                ? `评委与被测输出同厂商（${outv}）${calibrated ? '（已 gold 校准）' : ''}，存在自我偏好敞口：评委可能给同家族输出打高分`
+                : `Judge is the same vendor as the executor that produced the outputs (${outv})${calibrated ? ' (gold-calibrated)' : ''}; self-preference exposure — the judge may over-score same-family output`;
+        }
+        case 'single_vendor_ensemble': {
+            const n = asArray(details.judgeVendors).length;
+            const calibrated = details.goldCalibrated === true;
+            return lang === 'zh'
+                ? `${n} 个评委同属一个厂商${calibrated ? '（已 gold 校准）' : ''}，ensemble 一致性高只反映共有偏置，不构成对同模型偏置的反驳`
+                : `All ${n} judges are from one vendor${calibrated ? ' (gold-calibrated)' : ''}; high ensemble agreement reflects shared bias, not a rebuttal to same-model bias`;
+        }
         default:
             return lang === 'zh'
                 ? `结构化诊断：${type}`
@@ -1544,6 +1558,14 @@ function localizedSuggestion(insight, lang) {
             return lang === 'zh'
                 ? '重写 agent 断言时优先约束工具路径、关键文件读取和 turns 上限'
                 : 'When rewriting agent assertions, prioritize tool path, key file reads, and turn-limit constraints';
+        case 'judge_self_preference':
+            return lang === 'zh'
+                ? '换跨厂商评委（如 --judge-models openai-api:gpt-4o）消除自我偏好，或挂人工金标校准（omk eval gold compare）量化评委是否可信。注：固定模型的 A/B 差值受影响较小，绝对分 / 跨版本曲线受影响更大'
+                : 'Use a cross-vendor judge (e.g. --judge-models openai-api:gpt-4o) to remove self-preference, or calibrate against human gold (omk eval gold compare). Note: the A/B delta is less affected than absolute scores / cross-version curves';
+        case 'single_vendor_ensemble':
+            return lang === 'zh'
+                ? '把 ensemble 配成跨厂商（如 --judge-models claude:opus,openai-api:gpt-4o），让 agreement 真能反驳同模型偏置'
+                : 'Make the ensemble cross-vendor (e.g. --judge-models claude:opus,openai-api:gpt-4o) so agreement actually rebuts same-model bias';
         default:
             return lang === 'zh'
                 ? '查看结构化 details 字段定位原因'

package/dist/shared/llm-prompts/debias-instructions.d.ts

@@ -0,0 +1,4 @@
+export declare const LENGTH_DEBIAS_INSTRUCTION: string;
+export declare const PRESENTATION_NEUTRALITY_INSTRUCTION: string;
+export declare const RAG_LENGTH_DEBIAS: string;
+export declare const RAG_PRESENTATION_NEUTRALITY: string;

package/dist/shared/llm-prompts/debias-instructions.js

@@ -0,0 +1,44 @@
+// 评委 prompt 的共享去偏 / 中性化指令 —— 单一来源。
+//
+// 此前这几段在 grading/judge.ts(rubric 评委)与 grading/assertions.ts(RAG 评委)各写一份,
+// 改一处漏一处(J3 即如此)。统一收口于此,两侧 import。
+//
+// 两套变体是有意的、不要强并成一条:
+//   - 全版(LENGTH_DEBIAS_INSTRUCTION / PRESENTATION_NEUTRALITY_INSTRUCTION):给 rubric 评委,
+//     带研究依据句;rubric prompt 较短,容得下。被 getJudgePromptHash 冻结,改字节 = BREAKING-COMPARABILITY。
+//   - 短版(RAG_LENGTH_DEBIAS / RAG_PRESENTATION_NEUTRALITY):给 RAG 评委,RAG prompt 本就长,精简表述。
+// 改任一段先想另一段是否同步;改全版还要走冻结 hash 的 bump 流程。
+export const LENGTH_DEBIAS_INSTRUCTION = [
+    '## 重要：长度不是质量信号',
+    '评分时聚焦内容实质与正确性。回答的篇幅、行文丰富度、结构复杂度本身不是质量指标 ——',
+    '简洁正确的回答不应因短而扣分；冗长但偏题或重复的回答不应因长而加分。',
+    '研究显示 LLM 评委容易隐性偏向更长的回答，请在打分前先警觉这一点。',
+].join('\n');
+// 始终开启(不给开关):排版与语气都不是质量信号。措辞严格对称 —— 既不奖励精致 / 自信,也不
+// 因朴素 / 含糊而扣分,避免"抗偏置"本身过度矫正成反向偏置。研究表明 LLM 评委除长度外,还隐性
+// 偏向排版精致(format / markdown bias)与语气自信、自我表扬的回答(谄媚 / 权威偏置)。
+export const PRESENTATION_NEUTRALITY_INSTRUCTION = [
+    '## 重要：排版与语气不是质量信号',
+    '评分只看内容是否对照评分标准、是否正确。Markdown 排版、标题、列表、加粗、表格等呈现形式',
+    '本身不是质量指标 —— 朴素但正确的回答不应因没有排版而扣分；排版精致但偏题或错误的回答不应',
+    '因好看而加分。',
+    '同样，回答的语气、自信程度、是否自我表扬（如「这是最优方案」）也不是质量信号 —— 不要被笃定',
+    '的口吻或自我评价带跑：自信但错误的回答不应高于含糊但正确的回答，一切结论都要回到评分标准',
+    '逐条核实。',
+    '研究显示 LLM 评委容易隐性偏向排版精致、语气自信的回答，请在打分前先警觉这两点。',
+].join('\n');
+// RAG 指标(faithfulness / answer_relevancy / context_recall)的去偏**恒开**,
+// 按 default-strict 不提供关闭开关:`--no-debias-length` 只作用于 rubric 评委 prompt 的长度去偏,
+// 不下探到 RAG —— RAG 评的是内容保真 / 切题 / 召回,放任长度 / 排版 / 语气偏置只会污染这些指标,
+// 没有「关掉它」的正当用例。故 runRagJudge 不收 lengthDebias 参数,两段恒注入。
+export const RAG_LENGTH_DEBIAS = [
+    '## 重要：长度不是质量信号',
+    '评分时聚焦内容实质，不要因输出更长就给更高分。',
+    '简洁正确的回答与冗长正确的回答应得相同分数。',
+].join('\n');
+// 排版 / 语气中性化(format / sycophancy bias),与 RAG_LENGTH_DEBIAS 同 footprint 恒开。
+export const RAG_PRESENTATION_NEUTRALITY = [
+    '## 重要：排版与语气不是质量信号',
+    '评分只看内容是否符合上述标准。排版是否精致、语气是否自信、有无自我表扬都不是质量信号 ——',
+    '不要被笃定的口吻带跑：自信但错误的回答不应高于含糊但正确的回答。',
+].join('\n');

package/dist/shared/llm-prompts/judge-prompts.d.ts

@@ -0,0 +1,30 @@
+export declare const JUDGE_SYSTEM_PROMPT = "\u4F60\u662F\u4E00\u4E2A\u4E25\u683C\u7684 AI \u8F93\u51FA\u8D28\u91CF\u8BC4\u5BA1\u5458\u3002\u5148\u9010\u6761\u5BF9\u7167\u8BC4\u5206\u6807\u51C6\u505A\u63A8\u7406\uFF0C\u518D\u7ED9\u6700\u7EC8\u5206\u6570\u3002\u53EA\u8FD4\u56DE JSON\uFF0C\u4E0D\u8981\u5176\u4ED6\u5185\u5BB9\u3002";
+export declare function buildJudgePrompt(prompt: string, rubric: string, output: string, traceSummary: string | null, lengthDebias?: boolean): string;
+/**
+ * Stable hash of the judge prompt template. Saved into ReportMeta.judgePromptHash so
+ * downstream readers can detect "the judge prompt changed between these two reports".
+ *
+ * `lengthDebias` defaults to true. Pass false (via `--no-debias-length`) to drop the
+ * length-debias instruction; that produces the debias-off prompt variant, whose hash
+ * differs from the default so the two are never compared blind.
+ */
+export declare function getJudgePromptHash(lengthDebias?: boolean): string;
+export declare const SEMANTIC_SIMILARITY_SYSTEM = "\u4F60\u662F\u8BED\u4E49\u76F8\u4F3C\u5EA6\u8BC4\u5BA1\u5458\u3002\u53EA\u8FD4\u56DE JSON\uFF0C\u4E0D\u8981\u5176\u4ED6\u5185\u5BB9\u3002";
+export declare function buildSemanticSimilarityPrompt(reference: string, output: string): string;
+export type RagJudgeType = 'faithfulness' | 'answer_relevancy' | 'context_recall';
+export interface RagJudgeFields {
+    output: string;
+    /** faithfulness: 参考 context。 */
+    context?: string;
+    /** answer_relevancy: 用户问题(sample.prompt)。 */
+    question?: string;
+    /** context_recall: 参考 gold。 */
+    reference?: string;
+}
+/** RAG 评委 system + 用户 prompt。缺字段的早退 / 解析逻辑留在 assertions.ts。 */
+export declare function buildRagJudgePrompt(type: RagJudgeType, fields: RagJudgeFields): {
+    system: string;
+    prompt: string;
+};
+export declare function getSemanticPromptHash(): string;
+export declare function getRagJudgePromptHash(type: RagJudgeType): string;

package/dist/shared/llm-prompts/judge-prompts.js

@@ -0,0 +1,205 @@
+import { createHash } from 'node:crypto';
+import { LENGTH_DEBIAS_INSTRUCTION, PRESENTATION_NEUTRALITY_INSTRUCTION, RAG_LENGTH_DEBIAS, RAG_PRESENTATION_NEUTRALITY, } from './debias-instructions.js';
+// 评分类 prompt 单一来源 —— 直接决定分数的 LLM 评委 prompt（rubric 主评委 + RAG + 语义相似度）。
+// 这些是测量学不变量:文本字节决定可比性,改动须配合 prompt-registry 的冻结 hash bump
+// (BREAKING-COMPARABILITY)。执行器调用 / JSON 解析逻辑留在 grading/judge.ts 与 grading/assertions.ts。
+// ===========================================================================
+// Rubric 主评委 prompt
+// ===========================================================================
+/**
+ * Judge prompt template version.
+ *
+ * The version string encodes WHICH debias / context features the judge sees, so reports
+ * tagged with the same hash are score-comparable; a mismatched hash means "we changed how
+ * we ask the judge to think" and the reports should not be compared blind. Bump it (and the
+ * frozen hashes in `test/shared/prompt-registry-freeze.test.ts`) whenever the template's bytes
+ * change — that change is BREAKING-COMPARABILITY.
+ *
+ * Naming: a single main version (`v5`) + a `-feature` suffix per debias/context capability.
+ * The only asymmetry between the two strings is the `-len` suffix, gated by the
+ * `--no-debias-length` toggle; every other feature is always-on and appears in both.
+ */
+// 版本演进(内嵌"评委看到什么 / 被要求忽略什么"的语义):
+//   v2-cot               : 仅看 output + rubric + 工具名分布(早期 buildTraceSummary)
+//   v3-cot-length        : 加 length-debias 指令
+//   v3-cot-toolargs(off) / v4-cot-len-args(on)
+//                        : 加 tool input 预览(让评委识别 wrapper-style skill 在 Bash 命令里的
+//                          真实语义调用,如 `Bash: mcporter --tool X`,否则误判"只调了 Bash")
+//   v5-cot-toolargs-fmt(off) / v5-cot-toolargs-fmt-len(on)
+//                        : 加排版 / 语气中性化指令(始终开启,不给开关)。研究表明 LLM 评委除了
+//                          偏向更长的回答,还隐性偏向排版精致(标题 / 列表 / 加粗)与语气自信 / 自我
+//                          表扬(谄媚 / 权威偏置)的回答;显式指令要求评委只对照评分标准核内容。
+//                          同时借此把命名统一成单一主序号(v5)+ feature 后缀,`-len` 仍是开关那条。
+const JUDGE_PROMPT_VERSION_DEBIAS_OFF = 'v5-cot-toolargs-fmt';
+const JUDGE_PROMPT_VERSION_DEBIAS_ON = 'v5-cot-toolargs-fmt-len';
+export const JUDGE_SYSTEM_PROMPT = '你是一个严格的 AI 输出质量评审员。先逐条对照评分标准做推理，再给最终分数。只返回 JSON，不要其他内容。';
+export function buildJudgePrompt(prompt, rubric, output, traceSummary, lengthDebias = true) {
+    const version = lengthDebias ? JUDGE_PROMPT_VERSION_DEBIAS_ON : JUDGE_PROMPT_VERSION_DEBIAS_OFF;
+    const traceSection = traceSummary
+        ? ['', '## Agent 执行过程', traceSummary, '', '请同时考虑执行过程的合理性（工具选择、步骤效率、错误恢复）。']
+        : [];
+    // 排版 / 语气中性化始终开启(不受 --no-debias-length 影响);length-debias 仍受开关控。
+    const neutralitySection = ['', PRESENTATION_NEUTRALITY_INSTRUCTION];
+    const debiasSection = lengthDebias ? ['', LENGTH_DEBIAS_INSTRUCTION] : [];
+    return [
+        `请对以下 AI 输出进行质量评分（template ${version}）。`,
+        '',
+        '## 原始任务',
+        prompt,
+        '',
+        '## 评分标准',
+        rubric,
+        '',
+        '## AI 输出',
+        output,
+        ...traceSection,
+        ...neutralitySection,
+        ...debiasSection,
+        '',
+        '## 评分流程',
+        '1. 逐条对照评分标准，先做推理（reasoning）：列出 AI 输出哪些点对应哪条标准，哪些缺失，哪些有歧义。',
+        '2. 基于推理给出最终分数（1-5 的整数）和简短理由。',
+        '',
+        '请返回 JSON（不要包含 markdown 代码块标记）：',
+        '{"reasoning": "<对照标准的逐条推理>", "score": <1-5的整数>, "reason": "<最终结论的简短理由>"}',
+        '',
+        '评分标准：1=完全不达标, 2=部分涉及, 3=基本达标, 4=较好, 5=优秀',
+    ].join('\n');
+}
+/**
+ * Stable hash of the judge prompt template. Saved into ReportMeta.judgePromptHash so
+ * downstream readers can detect "the judge prompt changed between these two reports".
+ *
+ * `lengthDebias` defaults to true. Pass false (via `--no-debias-length`) to drop the
+ * length-debias instruction; that produces the debias-off prompt variant, whose hash
+ * differs from the default so the two are never compared blind.
+ */
+export function getJudgePromptHash(lengthDebias = true) {
+    const version = lengthDebias ? JUDGE_PROMPT_VERSION_DEBIAS_ON : JUDGE_PROMPT_VERSION_DEBIAS_OFF;
+    // Hash the template-shaping function source + the version tag together. We hash a
+    // deterministic stringified form of the template (with placeholder inputs) so any
+    // structural edit shows up.
+    const sample = buildJudgePrompt('<P>', '<R>', '<O>', '<T>', lengthDebias);
+    return createHash('sha256').update(version + '\n' + sample).digest('hex').slice(0, 12);
+}
+// ===========================================================================
+// 语义相似度评委 prompt（semantic_similarity 断言）
+// ===========================================================================
+export const SEMANTIC_SIMILARITY_SYSTEM = '你是语义相似度评审员。只返回 JSON，不要其他内容。';
+export function buildSemanticSimilarityPrompt(reference, output) {
+    return [
+        '请判断以下两段文本的语义相似度。',
+        '',
+        '## 参考文本',
+        reference,
+        '',
+        '## 待评估文本',
+        output,
+        '',
+        '请返回 JSON（不要包含 markdown 代码块标记）：',
+        '{"score": <1-5的整数>, "reason": "<简短理由>"}',
+        '',
+        '评分：1=完全无关, 2=略有关联, 3=部分相似, 4=大致相同, 5=高度一致',
+    ].join('\n');
+}
+/** RAG 评委 system + 用户 prompt。缺字段的早退 / 解析逻辑留在 assertions.ts。 */
+export function buildRagJudgePrompt(type, fields) {
+    if (type === 'faithfulness') {
+        return {
+            system: '你是 RAG 评审员,专注判断输出是否被参考 context 支持。只返回 JSON。',
+            prompt: [
+                '请判断"待评估输出"中的事实性陈述是否被"参考 context"支持。',
+                '',
+                '## 参考 context',
+                fields.context ?? '',
+                '',
+                '## 待评估输出',
+                fields.output,
+                '',
+                RAG_LENGTH_DEBIAS,
+                RAG_PRESENTATION_NEUTRALITY,
+                '',
+                '## 评分流程',
+                '1. 列出待评估输出中所有事实性陈述',
+                '2. 逐条判断是否能在 context 中找到支持',
+                '3. 给出 1-5 分:',
+                '   5 = 全部陈述都有 context 支持,无编造',
+                '   4 = 多数有支持,有 1-2 处不重要的编造',
+                '   3 = 一半有支持',
+                '   2 = 多数无支持',
+                '   1 = 完全编造或与 context 矛盾',
+                '',
+                '请返回 JSON(不要 markdown 代码块):',
+                '{"score": <1-5的整数>, "reason": "<简短理由>"}',
+            ].join('\n'),
+        };
+    }
+    if (type === 'answer_relevancy') {
+        return {
+            system: '你是答题切题度评审员。只返回 JSON。',
+            prompt: [
+                '请判断"AI 输出"是否直接、切题地回答了"用户问题"。',
+                '',
+                '## 用户问题',
+                fields.question ?? '',
+                '',
+                '## AI 输出',
+                fields.output,
+                '',
+                RAG_LENGTH_DEBIAS,
+                RAG_PRESENTATION_NEUTRALITY,
+                '',
+                '## 评分',
+                '5 = 完整切题回答,无冗余无遗漏',
+                '4 = 切题但有少量冗余或小遗漏',
+                '3 = 部分切题,部分跑题或避而不答',
+                '2 = 大部分跑题',
+                '1 = 完全跑题或拒答',
+                '',
+                '请返回 JSON(不要 markdown 代码块):',
+                '{"score": <1-5的整数>, "reason": "<简短理由>"}',
+            ].join('\n'),
+        };
+    }
+    // context_recall
+    return {
+        system: '你是 context 覆盖率评审员。只返回 JSON。',
+        prompt: [
+            '请判断"参考 gold"中的关键事实在"AI 输出"中被覆盖的程度。',
+            '',
+            '## 参考 gold',
+            fields.reference ?? '',
+            '',
+            '## AI 输出',
+            fields.output,
+            '',
+            RAG_LENGTH_DEBIAS,
+            RAG_PRESENTATION_NEUTRALITY,
+            '',
+            '## 评分流程',
+            '1. 列出参考中的关键事实(忽略修饰性内容)',
+            '2. 检查每条是否在输出中被提及/使用',
+            '3. 给出 1-5 分:',
+            '   5 = 全部关键事实被覆盖',
+            '   4 = 大部分覆盖,缺 1-2 条次要事实',
+            '   3 = 一半覆盖',
+            '   2 = 仅覆盖少量',
+            '   1 = 完全未覆盖',
+            '',
+            '请返回 JSON(不要 markdown 代码块):',
+            '{"score": <1-5的整数>, "reason": "<简短理由>"}',
+        ].join('\n'),
+    };
+}
+/** 评分类断言 prompt 的 drift-detection hash(占位符输入,12 位,仅供 prompt-registry 冻结)。 */
+function hashPromptSample(label, system, prompt) {
+    return createHash('sha256').update(`${label}\n${system}\n${prompt}`).digest('hex').slice(0, 12);
+}
+export function getSemanticPromptHash() {
+    return hashPromptSample('semantic_similarity', SEMANTIC_SIMILARITY_SYSTEM, buildSemanticSimilarityPrompt('<R>', '<O>'));
+}
+export function getRagJudgePromptHash(type) {
+    const fields = { output: '<O>', context: '<C>', question: '<Q>', reference: '<R>' };
+    const { system, prompt } = buildRagJudgePrompt(type, fields);
+    return hashPromptSample(`rag:${type}`, system, prompt);
+}

package/dist/shared/llm-prompts/registry.d.ts

@@ -0,0 +1,27 @@
+/**
+ * omk 所有 LLM prompt 的单一编目 —— 发现入口 + 测量学冻结入口。
+ *
+ * 解决「prompt 散在多个目录、难管理」与「只有 2 个 prompt 被冻结」两个问题:
+ *   - 每条登记 prompt 的位置(module)+ 用途,一处可查全。
+ *   - measurementInvariant 为 true 的(直接决定分数的评委 prompt)带 getHash,
+ *     由 test/shared/prompt-registry-freeze.test.ts 统一冻结,文本漂移即 CI 红。
+ *
+ * 注意:hash 形状按各自历史口径,不强行统一 ——
+ *   - rubric 评委:12 位(getJudgePromptHash,持久化进 report.meta.judgePromptHash 的同口径);
+ *   - RAG / semantic:12 位(getRagJudgePromptHash / getSemanticPromptHash,仅供冻结 drift 检测);
+ *   - observe 复盘:64 位(readPromptDocument 读 .md 全文 sha256)。
+ * 用限定名 `promptId`(非裸 `kind`,见 terminology-spec §5.4)。
+ */
+export interface PromptRegistryEntry {
+    /** 稳定标识(冻结表的 key)。 */
+    promptId: string;
+    /** 人类可读用途。 */
+    purpose: string;
+    /** 定义位置(发现用,相对仓库根的源文件路径)。 */
+    module: string;
+    /** 是否直接决定分数 → 须冻结防漂移。 */
+    measurementInvariant: boolean;
+    /** 仅 measurementInvariant 条目提供:当前 prompt 的 hash。 */
+    getHash?: () => string;
+}
+export declare const PROMPT_REGISTRY: PromptRegistryEntry[];