oh-my-knowledge 0.40.0 → 0.41.0

package/README.md

@@ -105,8 +105,7 @@ RAG-specific evals: see RAGAS (separate niche, complementary to omk). Full compa
 | **Git & remote sources** | install / eval from a local git ref or a remote git URL (`--git-url`); directory-skills run in a content-addressed **isolated copy** so `references/` assets are real measured input, not just `SKILL.md` |
 | **Evidence-gated management** | `omk install` registers a managed record; `omk eval` auto-writes evidence bound by content fingerprint, moving a skill `installed → measurable`; `omk list` surfaces each managed skill's status (installed / measurable / promoted / stale); `omk promote` accepts a version once its evidence passes the gate (default PROGRESS only); `omk rollback` revokes that acceptance, returning the skill to `measurable`. [spec →](docs/specs/evidence-gated-management.md) |
 | **Sample design science** | sample schema with `capability` / `difficulty` / `construct` / `provenance` metadata (HF Dataset Cards style); studio surfaces coverage breakdown plus `rubric_clarity_low` / `capability_thin` flags. [docs/specs/sample-design-spec.md](docs/specs/sample-design-spec.md) |
-| **Multi-judge ensemble** | `--judge-models claude:opus,openai:gpt-4o` cross-vendor scoring + agreement metrics |
-| **Blind A/B** | `--blind` hides variant names; HTML report has a reveal button |
+| **Multi-judge ensemble** | `--judge-models claude:opus,openai-api:gpt-4o` cross-vendor scoring + agreement metrics |
 | **Multi-run variance** | `--repeat N` repeats the eval and computes mean / SD / CI / t-test |
 | **MCP URL fetching** | pull content from private-doc URLs via an MCP server (SSO-protected knowledge bases, etc.) |
 | **Auto analysis** | detects low-discrimination assertions, flat scores, all-pass / all-fail, expensive samples |

package/README.zh.md

@@ -105,8 +105,7 @@ RAG 专项评测请看 RAGAS（独立 niche，跟 omk 互补）。完整对比
 | **Git / 远端源** | install / eval 支持本地 git ref 或远端 git URL（`--git-url`）；目录-skill 在内容寻址**隔离副本**里执行，`references/` 资产是真实测量输入，不只是 `SKILL.md` |
 | **证据门控管理** | `omk install` 登记受管记录；`omk eval` 按内容指纹自动写入证据，把 skill 从 `installed` 推到 `measurable`；`omk list` 查看各受管 skill 的状态（installed / measurable / promoted / stale）；`omk promote` 在证据过门禁（默认仅 PROGRESS）后把该版本接受为当前版本；`omk rollback` 撤销这次接受，让 skill 回到 `measurable`。[规范 →](docs/zh/specs/evidence-gated-management.md) |
 | **用例设计科学性** | Sample schema 加 `capability` / `difficulty` / `construct` / `provenance` 元数据字段（HF Dataset Cards 风），studio 输出 coverage 分桶 + `rubric_clarity_low` / `capability_thin` issue。[docs/zh/specs/sample-design-spec.md](docs/zh/specs/sample-design-spec.md) |
-| **多评委 ensemble** | `--judge-models claude:opus,openai:gpt-4o` 跨厂商评分 + agreement 度量 |
-| **盲测 A/B** | `--blind` 隐藏变体名称，HTML 报告有揭晓按钮 |
+| **多评委 ensemble** | `--judge-models claude:opus,openai-api:gpt-4o` 跨厂商评分 + agreement 度量 |
 | **多轮方差分析** | `--repeat N` 重复 N 次，计算均值/标准差/置信区间/t 检验 |
 | **MCP URL 获取** | 通过 MCP Server 获取私有文档 URL 内容（SSO 保护的知识库等） |
 | **自动分析** | 检测低区分度断言、均匀分数、全通过/全失败、高成本用例 |

package/dist/analysis/report-diagnostics.js

@@ -2,6 +2,7 @@
  * Auto-analysis: detect patterns and generate insights from evaluation results.
  */
 import { normalizeCapability } from './sample-diagnostics.js';
+import { analyzeJudgeIndependence } from '../eval-core/judge-independence.js';
 /**
  * Analyze an evaluation report and produce structured insights.
  */
@@ -15,6 +16,10 @@ export function analyzeResults(report, opts = {}) {
     const sampleQuality = opts.samples
         ? buildSampleQualityAggregate(opts.samples)
         : undefined;
+    // 评委独立性对**单变体**报告同样适用(SOLO 只有绝对分、没有 A/B 差值去抵消自我偏好,
+    // 反而更该报),故放在 variants<2 早退之前;仅 results 为空(dry-run)时不评估。
+    if (results.length > 0)
+        detectJudgeIndependence(report, insights);
     if (results.length === 0 || variants.length < 2) {
         return { insights, ...(sampleQuality && { sampleQuality }) };
     }
@@ -38,11 +43,33 @@ export function analyzeResults(report, opts = {}) {
     detectAgentAssertionDiscrimination(results, variants, insights);
     // 10. Suggest --repeat when score variance is high and no repeat data
     detectNeedRepeat(report, results, variants, insights);
+    // 11. Judge independence — 已在 variants<2 早退前调过(单变体也适用),此处不重复。
     return {
         insights,
         ...(sampleQuality && { sampleQuality }),
     };
 }
+/**
+ * 评委独立性诊断:同厂商评委(自我偏好敞口)/ 单厂商 ensemble(一致性不反驳共有偏置)。
+ * 判定走单一来源 `analyzeJudgeIndependence`;挂了 gold 校准则软化为 info。
+ */
+function detectJudgeIndependence(report, insights) {
+    const ind = analyzeJudgeIndependence(report);
+    if (ind.sameVendorJudge) {
+        insights.push({
+            type: 'judge_self_preference',
+            severity: ind.goldCalibrated ? 'info' : 'warning',
+            details: { judgeVendors: ind.judgeVendors, outputVendors: ind.outputVendors, goldCalibrated: ind.goldCalibrated },
+        });
+    }
+    if (ind.singleVendorEnsemble) {
+        insights.push({
+            type: 'single_vendor_ensemble',
+            severity: ind.goldCalibrated ? 'info' : 'warning',
+            details: { judgeVendors: ind.judgeVendors, goldCalibrated: ind.goldCalibrated },
+        });
+    }
+}
 /**
  * Build sample design science aggregate from sample metadata.
  *

package/dist/assets/agent-skills/omk/references/commands.md

@@ -67,7 +67,6 @@ omk eval [flags]
 **Flags:**
 
 - `--batch` `boolean`:batch 模式:baseline vs 每个 skill
-- `--blind` `boolean`:judge blind 模式
 - `--bootstrap` `boolean`:加 bootstrap CI
 - `--bootstrap-samples` `option`:bootstrap 重采样次数，默认 1000
 - `--budget-per-sample-ms` `option`:单 sample 时长上限 ms（必须 > 0，不传则无上限）
@@ -82,7 +81,7 @@ omk eval [flags]
 - `--executor` `option`:执行器:claude / claude-sdk / codex / codex-sdk / openai-api / gemini / 自定义命令（默认 claude）。
 - `--global` `boolean`:报告写全局 ~/.oh-my-knowledge/reports，而非项目 .omk/
 - `--gold-dir` `option`:gold dataset 目录
-- `--judge-models` `option`:评委配置，格式 executor:model[,...]，例 claude:haiku 或 claude:opus,openai:gpt-4o(≥ 2 个 = ensemble）。默认 <executor>:haiku。
+- `--judge-models` `option`:评委配置，格式 executor:model[,...]，例 claude:haiku 或 claude:opus,openai-api:gpt-4o(≥ 2 个 = ensemble）。默认 <executor>:haiku。
 - `--judge-repeat` `option`:每个 dim 评 N 次
 - `--lang` `option` (默认 `zh`):输出语言 zh|en，优先级 CLI > OMK_LANG env > zh。
 - `--layered-stats` `boolean`:输出分层统计

package/dist/authoring/generator.js

@@ -1,4 +1,5 @@
 import { createExecutor } from '../executors/index.js';
+import { DEFAULT_GATE_THRESHOLD } from '../eval-core/verdict.js';
 /**
  * Generator 默认模型 'opus' (跟 eval 默认对齐)。
  * lean=true 路径会自动追加 `--effort low`,关掉 opus 默认的扩展思考,
@@ -76,7 +77,7 @@ const SYSTEM_PROMPT = `你是一个评测用例生成器。你的任务是根据
   来涨数量)。omk 的评分体系是 layered scoring: **fact 层**(deterministic 字面/工具断言)
   + **behavior 层**(代价指标如 turn 数 / 工具失败率) + **judge 层**(主观语义评分,从
   sample.rubric 派生维度,judge LLM 看 trace 评 1-5)三层独立计分,verdict 是三层
-  独立过 threshold(默认 3.5)。**fact 层的本职是测 deterministic 端点,不是测轨迹**。
+  独立过 threshold(默认 ${DEFAULT_GATE_THRESHOLD})。**fact 层的本职是测 deterministic 端点,不是测轨迹**。
 
   **断言哲学(关键):fact 测结果+里程碑,过程质量交 judge**
   ─────────────────────────────────────────────────────────────
@@ -109,7 +110,7 @@ const SYSTEM_PROMPT = `你是一个评测用例生成器。你的任务是根据
     - tools_not_called 反模式断言 0-1 条(禁止接触某禁忌工具,如 tripwire sample)
     - rubric 3-5 个判分维度(细致写明 judge 该看什么),由 sample.rubric 字段承载
 
-  *测量学背景:* 当前 omk verdict 三层独立 threshold(默认 3.5),fact 条目少之后单条
+  *测量学背景:* 当前 omk verdict 三层独立 threshold(默认 ${DEFAULT_GATE_THRESHOLD}),fact 条目少之后单条
   权重大、单次评测方差大,**强烈建议** 评测时带 \`--repeat 2\` 或更大测稳定性(coefficient
   of variation),并参考 bootstrap CI 而非点估计。这是 fact 层稀疏化的代价,换来的是
   fact 信号干净(不被 trajectory 字面噪音污染)。

package/dist/cli/commands/eval/index.d.ts

@@ -37,7 +37,6 @@ export default class Eval extends BaseCommand {
         'no-strict-baseline': import("@oclif/core/interfaces").BooleanFlag<boolean>;
         effort: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
         'no-diagnostic': import("@oclif/core/interfaces").BooleanFlag<boolean>;
-        blind: import("@oclif/core/interfaces").BooleanFlag<boolean>;
         repeat: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
         'judge-repeat': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
         bootstrap: import("@oclif/core/interfaces").BooleanFlag<boolean>;

package/dist/cli/commands/eval/index.js

@@ -8,6 +8,7 @@ import { parseRunConfig } from '../../lib/parse-run-config.js';
 import { makeOnProgress } from '../../lib/progress.js';
 import { computeRunTally } from '../../lib/run-tally.js';
 import { DEFAULT_BOOTSTRAP_SAMPLES } from '../../../eval-core/bootstrap.js';
+import { DEFAULT_GATE_THRESHOLD } from '../../../eval-core/verdict.js';
 import { EVALUATION_REPORT_SCHEMA_VERSION } from '../../../eval-core/evaluation-reporting.js';
 function isDryRunReport(report) {
     return Boolean(report && typeof report === 'object' && report.dryRun === true);
@@ -17,9 +18,11 @@ function isDryRunBatchReport(report) {
 }
 function verdictOptions(values) {
     const rawThreshold = values.threshold;
+    // 不传 --threshold 时返回 undefined,由 computeVerdict 应用 DEFAULT_GATE_THRESHOLD ——
+    // 避免在此再硬编码一份 3.5(单一来源在 verdict.ts)。
     const gateThreshold = rawThreshold !== undefined && Number.isFinite(Number(rawThreshold))
         ? Number(rawThreshold)
-        : 3.5;
+        : undefined;
     const rawTrivial = values['trivial-diff'];
     const triviallySmallDiff = rawTrivial !== undefined && Number.isFinite(Number(rawTrivial))
         ? Number(rawTrivial)
@@ -158,9 +161,6 @@ async function announceSavedReport({ report, filePath, reportsDir, values, lang,
 async function runEval(_args, flags, lang) {
     const { values, config, evalConfig } = parseRunConfig({ ...flags });
     const { runEvaluation, runMultiple, runBatchEvaluation } = await import('../../../eval-workflows/run-evaluation.js');
-    if (values.blind !== undefined) {
-        config.blind = values.blind;
-    }
     config.onProgress = makeOnProgress(lang);
     const repeatRaw = values.repeat;
     const parsedRepeat = repeatRaw !== undefined ? Number(repeatRaw) : (evalConfig?.repeat ?? 1);
@@ -368,8 +368,8 @@ export default class Eval extends BaseCommand {
         }),
         'judge-models': Flags.string({
             description: bilingual({
-                zh: '评委配置，格式 executor:model[,...]，例 claude:haiku 或 claude:opus,openai:gpt-4o(≥ 2 个 = ensemble）。默认 <executor>:haiku。',
-                en: 'Judge config: executor:model[,...]. e.g. claude:haiku or claude:opus,openai:gpt-4o (≥ 2 = ensemble). Default <executor>:haiku.',
+                zh: '评委配置，格式 executor:model[,...]，例 claude:haiku 或 claude:opus,openai-api:gpt-4o(≥ 2 个 = ensemble）。默认 <executor>:haiku。',
+                en: 'Judge config: executor:model[,...]. e.g. claude:haiku or claude:opus,openai-api:gpt-4o (≥ 2 = ensemble). Default <executor>:haiku.',
             }),
         }),
         'output-dir': Flags.string({
@@ -453,9 +453,6 @@ export default class Eval extends BaseCommand {
             }),
         }),
         // ── eval-runner extra ──
-        blind: Flags.boolean({
-            description: bilingual({ zh: 'judge blind 模式', en: 'Blind judge mode' }),
-        }),
         repeat: Flags.string({
             description: bilingual({ zh: '每个 sample 重复跑 N 次', en: 'Repeat each sample N times' }),
             parse: integerStringParser('--repeat', { min: 1 }),
@@ -490,7 +487,10 @@ export default class Eval extends BaseCommand {
             parse: numberStringParser('--budget-per-sample-ms', { minExclusive: 0 }),
         }),
         threshold: Flags.string({
-            description: bilingual({ zh: 'verdict 阈值，默认 3.5', en: 'Verdict threshold, default 3.5' }),
+            description: bilingual({
+                zh: `verdict 阈值，默认 ${DEFAULT_GATE_THRESHOLD}`,
+                en: `Verdict threshold, default ${DEFAULT_GATE_THRESHOLD}`,
+            }),
             parse: numberStringParser('--threshold'),
         }),
         'trivial-diff': Flags.string({

package/dist/cli/lib/cmd-flags.d.ts

@@ -96,7 +96,6 @@ export interface EvalFlags {
     'no-strict-baseline'?: boolean;
     effort?: string;
     'no-diagnostic'?: boolean;
-    blind?: boolean;
     repeat?: string;
     'judge-repeat'?: string;
     bootstrap?: boolean;

package/dist/cli/lib/i18n-dict/run.js

@@ -52,8 +52,8 @@ export const runDict = {
         en: '⚠ --judge-repeat "{value}" is invalid (expected an integer ≥ 1), falling back to 1 judge call\n',
     },
     'cli.run.no_debias_length_active': {
-        zh: 'ℹ --no-debias-length 已生效: judge prompt 退回 v2-cot, 与 < v0.21 报告 hash 一致。\n',
-        en: 'ℹ --no-debias-length is active: judge prompt reverts to v2-cot, matching < v0.21 report hashes.\n',
+        zh: 'ℹ --no-debias-length 已生效：judge prompt 去掉长度去偏指令（debias-off 变体），hash 与默认开启时不同。\n',
+        en: 'ℹ --no-debias-length is active: the judge prompt drops the length-debias instruction (debias-off variant); its hash differs from the default.\n',
     },
     'cli.run.invalid_bootstrap_samples': {
         zh: '⚠ --bootstrap-samples "{value}" 无效 (期望 ≥ 100 的整数), 已按 1000 执行\n',

package/dist/cli/lib/parse-run-config.d.ts

@@ -42,7 +42,6 @@ export interface RunConfig {
     lang: 'zh' | 'en' | undefined;
     mcpConfig: string | undefined;
     verbose: boolean | undefined;
-    blind?: boolean | undefined;
     retry?: number;
     resume?: string;
     layeredStats?: boolean;

package/dist/cli/lib/parse-run-config.js

@@ -94,7 +94,6 @@ export function parseRunConfig(values) {
     const verbose = values.verbose ?? false;
     const retry = Math.max(0, Number(values.retry ?? 0) || 0);
     const resume = values.resume;
-    const blind = values.blind ?? evalConfig?.blind ?? false;
     const layeredStats = values['layered-stats'] ?? false;
     // strict-baseline default true. Reconcile both flag forms with eval.yaml fallback.
     // Priority: --no-strict-baseline > --strict-baseline > eval.yaml strictBaseline > true。
@@ -144,7 +143,6 @@ export function parseRunConfig(values) {
             verbose,
             retry,
             resume,
-            blind,
             layeredStats,
             budget: evalConfig?.budget,
             strictBaseline,

package/dist/eval-core/evaluation-job.d.ts

@@ -1,5 +1,5 @@
 import type { Artifact, EvaluationErrorCategory, EvaluationJob, EvaluationRequest, EvaluationRun, JudgeConfig } from '../types/index.js';
-export declare function buildEvaluationRequest({ samplesPath, skillDir, artifacts, model, executor, noJudge, concurrency, timeoutMs, noCache, dryRun, blind, project, owner, tags, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }: {
+export declare function buildEvaluationRequest({ samplesPath, skillDir, artifacts, model, executor, noJudge, concurrency, timeoutMs, noCache, dryRun, project, owner, tags, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }: {
     samplesPath: string;
     skillDir: string;
     artifacts: Artifact[];
@@ -10,7 +10,6 @@ export declare function buildEvaluationRequest({ samplesPath, skillDir, artifact
     timeoutMs?: number;
     noCache: boolean;
     dryRun: boolean;
-    blind: boolean;
     project?: string;
     owner?: string;
     tags?: string[];

package/dist/eval-core/evaluation-job.js

@@ -1,7 +1,7 @@
 function nowIso() {
     return new Date().toISOString();
 }
-export function buildEvaluationRequest({ samplesPath, skillDir, artifacts, model, executor, noJudge, concurrency, timeoutMs, noCache, dryRun, blind, project, owner, tags, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }) {
+export function buildEvaluationRequest({ samplesPath, skillDir, artifacts, model, executor, noJudge, concurrency, timeoutMs, noCache, dryRun, project, owner, tags, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }) {
     return {
         samplesPath,
         skillDir,
@@ -13,7 +13,6 @@ export function buildEvaluationRequest({ samplesPath, skillDir, artifacts, model
         timeoutMs,
         noCache,
         dryRun,
-        blind,
         project,
         owner,
         tags,

package/dist/eval-core/evaluation-reporting.d.ts

@@ -28,7 +28,6 @@ interface AggregateReportOptions {
     layeredStats?: boolean;
 }
 export declare function aggregateReport({ runId, variants, model, judgeModel, noJudge, executorName, samples, tasks, results, totalCostUSD, artifacts, request, run, job, layeredStats, }: AggregateReportOptions): Report;
-export declare function applyBlindMode(report: Report, variants: string[], blindSeed: string): void;
 export interface PersistableReport {
     id: string;
 }

package/dist/eval-core/evaluation-reporting.js

@@ -199,8 +199,8 @@ export function aggregateReport({ runId, variants, model, judgeModel, noJudge, e
         ...(noJudge ? {} : { runtime: getExecutorRuntimeFingerprint(jc.executor, jc.model, runtimeOptions) }),
     }));
     // length-debias is on by default; the request only sets it
-    // false when the user passed --no-debias-length. The hash differs between
-    // v3-cot-length (on) and v2-cot (off) so readers can detect the divergence.
+    // false when the user passed --no-debias-length. The judgePromptHash differs between
+    // the length-debias-on and -off prompt variants so readers can detect the divergence.
     const lengthDebiasOn = request?.lengthDebias !== false;
     const debiasModeList = [];
     if (lengthDebiasOn)
@@ -271,42 +271,6 @@ export function aggregateReport({ runId, variants, model, judgeModel, noJudge, e
             }])),
     };
 }
-export function applyBlindMode(report, variants, blindSeed) {
-    const labels = variants.map((_, i) => String.fromCharCode(65 + i));
-    let seed = parseInt(hashString(blindSeed).slice(0, 8), 16) | 0;
-    const seededRandom = () => {
-        seed |= 0;
-        seed = seed + 0x6D2B79F5 | 0;
-        let value = Math.imul(seed ^ seed >>> 15, 1 | seed);
-        value ^= value + Math.imul(value ^ value >>> 7, 61 | value);
-        return ((value ^ value >>> 14) >>> 0) / 4294967296;
-    };
-    const shuffled = [...variants];
-    for (let i = shuffled.length - 1; i > 0; i--) {
-        const j = Math.floor(seededRandom() * (i + 1));
-        [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
-    }
-    const blindMap = Object.fromEntries(shuffled.map((variant, i) => [labels[i], variant]));
-    const reverseMap = Object.fromEntries(Object.entries(blindMap).map(([label, variant]) => [variant, label]));
-    report.meta.blind = true;
-    report.meta.blindMap = blindMap;
-    report.meta.variants = labels;
-    if (report.meta.executorRuntimes) {
-        report.meta.executorRuntimes = Object.fromEntries(Object.entries(report.meta.executorRuntimes).map(([variant, runtime]) => [reverseMap[variant] ?? variant, runtime]));
-    }
-    const newSummary = {};
-    for (const [variant, stats] of Object.entries(report.summary)) {
-        newSummary[reverseMap[variant]] = stats;
-    }
-    report.summary = newSummary;
-    for (const result of report.results) {
-        const newVariants = {};
-        for (const [variant, data] of Object.entries(result.variants)) {
-            newVariants[reverseMap[variant]] = data;
-        }
-        result.variants = newVariants;
-    }
-}
 export function persistReport(report, outputDir) {
     if (!outputDir)
         return null;

package/dist/eval-core/execution-strategy.js

@@ -101,8 +101,9 @@ export function resolveExecutionStrategy(task, model, timeoutMs, verbose, effort
         verbose,
         ...(effort && { effort }),
         // pass skill-isolation declaration to executors. undefined keeps
-        // SDK default; [] = strict isolation (skills:[] + disallowedTools:['Skill']);
-        // [...] = whitelist (skills:[...] only).
+        // SDK default; [] = strict isolation (skills:[] + disallowedTools:['Skill']).
+        // non-empty allowedSkills is rejected upstream (validateEvalConfig) and by every
+        // executor — a skill whitelist could not be fully isolated, so it was removed.
         ...(task.artifact.allowedSkills !== undefined && { allowedSkills: task.artifact.allowedSkills }),
         // Sample.mocks 透传到 executor。executor(claude-sdk / claude-cli)
         // 自决定怎么落地(in-process hook vs 临时 CLAUDE_CONFIG_DIR + on-disk hook)。

package/dist/eval-core/judge-independence.d.ts

@@ -0,0 +1,28 @@
+import type { Report } from '../types/index.js';
+import { type ExecutorVendor } from '../executors/shared.js';
+/**
+ * 评委独立性分析(单一来源,verdict caveat 与 analysis 诊断共用)。
+ *
+ * LLM 评委有自我偏好偏置:偏爱与自己同模型家族产出的输出。omk 默认评委(claude:haiku)与默认
+ * 执行器(claude:*)同属一家,敞口默认就开着。本 helper 只算客观事实(评委/输出各属哪家、有没有
+ * 跨厂商评委、是不是单厂商 ensemble、有没有挂 gold 校准),严重度与文案交给消费方(report-diagnostics
+ * / verdict)决定。
+ *
+ * 缓解阶梯(行业共识):换跨厂商评委 > 跨厂商陪审团 > 人工金标校准 > 警告。omk 无法强制换 key,
+ * 故只检测 + 警告并指向 `--judge-models <跨厂商>` 和 `omk eval gold compare`。
+ */
+export interface JudgeIndependence {
+    /** 各评委的厂商家族(与 judgeModels 同序)。 */
+    judgeVendors: ExecutorVendor[];
+    /** 被测输出涉及的厂商家族(去重)。 */
+    outputVendors: ExecutorVendor[];
+    /** 至少有一个评委的厂商不在被测输出厂商集合里 —— 存在独立(跨厂商)评委。 */
+    crossVendorJudgePresent: boolean;
+    /** 有评委、厂商全可归类、且无任何跨厂商评委 —— 自我偏好敞口(J1)。 */
+    sameVendorJudge: boolean;
+    /** ≥ 2 评委且全部同一个(已归类)厂商 —— ensemble 一致性不反驳共有偏置(J2)。 */
+    singleVendorEnsemble: boolean;
+    /** 本次 run 挂了人工 gold 校准(report.meta.humanAgreement 存在)→ 敞口有背板。 */
+    goldCalibrated: boolean;
+}
+export declare function analyzeJudgeIndependence(report: Report): JudgeIndependence;

package/dist/eval-core/judge-independence.js

@@ -0,0 +1,29 @@
+import { executorVendor } from '../executors/shared.js';
+export function analyzeJudgeIndependence(report) {
+    const judges = report.meta?.judgeModels ?? [];
+    const judgeVendors = judges.map((j) => executorVendor(j.executor));
+    const runtimes = report.meta?.executorRuntimes;
+    const outputExecutors = runtimes && Object.keys(runtimes).length > 0
+        ? Object.values(runtimes).map((r) => r.executor).filter((e) => !!e)
+        : (report.meta?.executor ? [report.meta.executor] : []);
+    const outputVendors = [...new Set(outputExecutors.map(executorVendor))];
+    const goldCalibrated = report.meta?.humanAgreement != null;
+    const base = { judgeVendors, outputVendors, goldCalibrated };
+    // 不评估(不误报)的情形:
+    //   - noJudge:judgeModels 为审计保留(非空),但评委根本没跑、composite 无评委层 → 自我偏好 moot;
+    //   - 无评委 / 任一厂商无法归类(自定义 script)/ 拿不到被测厂商。
+    const noJudge = report.meta?.noJudge === true;
+    const anyUnknown = judgeVendors.includes('unknown') || outputVendors.includes('unknown');
+    if (judges.length === 0 || noJudge || anyUnknown || outputVendors.length === 0) {
+        return { ...base, crossVendorJudgePresent: false, sameVendorJudge: false, singleVendorEnsemble: false };
+    }
+    const outSet = new Set(outputVendors);
+    const crossVendorJudgePresent = judgeVendors.some((v) => !outSet.has(v));
+    const singleVendorEnsemble = judges.length >= 2 && new Set(judgeVendors).size === 1;
+    return {
+        ...base,
+        crossVendorJudgePresent,
+        sameVendorJudge: !crossVendorJudgePresent,
+        singleVendorEnsemble,
+    };
+}

package/dist/eval-core/verdict.d.ts

@@ -55,6 +55,14 @@ export declare const ENSEMBLE_DISSENT_PEARSON = 0.4;
  * never gated by it — see `computeVerdict`.
  */
 export declare const STABILITY_UNSTABLE_CV = 0.15;
+/**
+ * Per-layer pass/fail line for the three-layer gate, on the 1-5 scale.
+ * **Pragmatic default, not derived from an external standard**: 3.5 is a clear margin
+ * above the 3.0 scale midpoint ("basically acceptable"), so a layer must land
+ * comfortably in the upper half to pass. Overridable via `omk eval --threshold`.
+ * doc ↔ code parity guarded by `test/scripts/doc-constants-drift.test.ts`.
+ */
+export declare const DEFAULT_GATE_THRESHOLD = 3.5;
 export type VerdictLevel = 'PROGRESS' | 'CAUTIOUS' | 'REGRESS' | 'NOISE' | 'UNDERPOWERED' | 'SOLO';
 export interface VerdictResult {
     level: VerdictLevel;
@@ -82,7 +90,7 @@ export interface VerdictResult {
     variants: string[];
 }
 export interface VerdictOptions {
-    /** Three-layer ci-gate threshold; defaults to 3.5 (matches `omk eval`). */
+    /** Three-layer ci-gate threshold; defaults to DEFAULT_GATE_THRESHOLD (matches `omk eval`). */
     gateThreshold?: number;
     /**
      * Magnitude (in raw score points) below which a "significant" diff is treated

package/dist/eval-core/verdict.js

@@ -29,6 +29,7 @@
  */
 import { evaluateLayerGates } from './layer-gates.js';
 import { ciLevelLabel } from './bootstrap.js';
+import { analyzeJudgeIndependence } from './judge-independence.js';
 /**
  * Below this sample count a non-significant diff is read as UNDERPOWERED
  * (only large effects are detectable) rather than NOISE. Matches the
@@ -56,11 +57,19 @@ export const ENSEMBLE_DISSENT_PEARSON = 0.4;
  * never gated by it — see `computeVerdict`.
  */
 export const STABILITY_UNSTABLE_CV = 0.15;
+/**
+ * Per-layer pass/fail line for the three-layer gate, on the 1-5 scale.
+ * **Pragmatic default, not derived from an external standard**: 3.5 is a clear margin
+ * above the 3.0 scale midpoint ("basically acceptable"), so a layer must land
+ * comfortably in the upper half to pass. Overridable via `omk eval --threshold`.
+ * doc ↔ code parity guarded by `test/scripts/doc-constants-drift.test.ts`.
+ */
+export const DEFAULT_GATE_THRESHOLD = 3.5;
 /**
  * Compute a verdict for a finished report. Pure function — no I/O.
  */
 export function computeVerdict(report, options = {}) {
-    const { gateThreshold = 3.5, triviallySmallDiff = 0.1 } = options;
+    const { gateThreshold = DEFAULT_GATE_THRESHOLD, triviallySmallDiff = 0.1 } = options;
     const variants = report.meta?.variants ?? [];
     const summary = report.summary ?? {};
     const sampleCount = report.meta?.sampleCount ?? 0;
@@ -68,15 +77,18 @@ export function computeVerdict(report, options = {}) {
         // Single-variant — no comparison possible. Just report whether the variant
         // passes its own three-layer gate.
         const gate = evaluateLayerGates(summary, gateThreshold);
+        // SOLO 只有绝对分、无 A/B 差值可抵消自我偏好,故同厂商评委的 caveat 更该出。
+        const judgeInd = judgeIndependenceCaveat(report);
         return {
             level: 'SOLO',
-            headline: gate.allPass
+            headline: (gate.allPass
                 ? `SOLO · single variant, three-layer gate PASS @ threshold ${gateThreshold}`
-                : `SOLO · single variant, three-layer gate FAIL — see ci output`,
+                : `SOLO · single variant, three-layer gate FAIL — see ci output`) + judgeInd.note,
             rationale: {
                 layerWinners: gate.lines.join('; '),
                 sampleSize: `N=${sampleCount}`,
                 stability: formatStability(report),
+                ...(judgeInd.rationale ? { judgeAgreement: judgeInd.rationale } : {}),
             },
             variants,
         };
@@ -110,18 +122,19 @@ export function computeVerdict(report, options = {}) {
     const stabilityNote = stabilityGated && stab
         ? ` · 显著但 run-to-run 不稳(CV=${(stab.cv * 100).toFixed(1)}% > ${(STABILITY_UNSTABLE_CV * 100).toFixed(0)}%)`
         : '';
+    const judgeInd = judgeIndependenceCaveat(report);
     const significance = representative
         ? formatSignificance(representative)
         : 'no pairwise comparison available — was --bootstrap used?';
     const layerWinners = formatLayerWinners(summary, variants);
     const sampleSize = formatSampleSize(report);
     const stability = formatStability(report);
-    const judgeAgreement = formatJudgeAgreement(report);
+    const judgeAgreement = [formatJudgeAgreement(report), judgeInd.rationale].filter(Boolean).join(' · ') || undefined;
     const shipRecommendation = recommendation(level, perPair);
     return {
         level,
         headline: representative
-            ? `${level} · ${representative.treatment} vs ${representative.control}: ${representative.headline}${stabilityNote}`
+            ? `${level} · ${representative.treatment} vs ${representative.control}: ${representative.headline}${stabilityNote}${judgeInd.note}`
             : `${level} · ${variants.length} variants`,
         perPair,
         rationale: {
@@ -376,6 +389,31 @@ function formatJudgeAgreement(report) {
                     : 'poor';
     return `α=${Number.isNaN(a.alpha) ? 'NaN' : a.alpha.toFixed(2)} (${verdict}) vs gold ${a.goldAnnotator}`;
 }
+/**
+ * 评委独立性 caveat(自我偏好 J1 / 单厂商 ensemble J2)。返回 { note, rationale }:
+ *   note      —— 追到 headline 的短提示(仅未 gold 校准时出,提醒读者敞口);
+ *   rationale —— 并进 rationale.judgeAgreement 的可执行说明(指向跨厂商评委 / gold)。
+ * **不改 verdict level**:omk 固定模型,自我偏好对 baseline / treatment 两臂同等加成、在 verdict
+ * 在意的 A/B 差值里大幅抵消,不该翻 ship/no-ship;真正受影响的是绝对分 / 版本曲线 / 跨模型比较。
+ */
+function judgeIndependenceCaveat(report) {
+    const ind = analyzeJudgeIndependence(report);
+    const reasons = [];
+    if (ind.sameVendorJudge)
+        reasons.push(`评委与被测输出同厂商(${ind.outputVendors.join('/')})`);
+    if (ind.singleVendorEnsemble)
+        reasons.push(`${ind.judgeVendors.length} 个评委同厂商，ensemble 一致性不反驳同模型偏置`);
+    if (reasons.length === 0)
+        return { note: '' };
+    if (ind.goldCalibrated) {
+        // 有 gold 校准背板 → 不进 headline,只软提示。
+        return { note: '', rationale: `自我偏好敞口（${reasons.join('；')}）已有 gold 校准背板` };
+    }
+    return {
+        note: ' · 评委自我偏好敞口未校准',
+        rationale: `${reasons.join('；')} —— 绝对分可能偏高；换跨厂商评委(--judge-models)或挂 gold(omk eval gold compare)校准`,
+    };
+}
 function recommendation(level, _perPair) {
     switch (level) {
         case 'PROGRESS':

package/dist/eval-workflows/batch-evaluation-workflow.d.ts

@@ -52,7 +52,7 @@ interface CompletedBatchSkillRun {
  *  导致 #183 角色误绑各存一份)。
  *  两个 variant 的 allowedSkills 都从 eval.yaml variants[].allowedSkills 取:treatment 按 skill
  *  名 entry.name 查、baseline 按保留名 `baseline` 查,挂到对应 spec 上由 prepareEvaluationRun
- *  统一绑定。baseline 的显式声明(白名单或 `[]`)必须保留——eval.yaml variant.allowedSkills
+ *  统一绑定。baseline 的显式声明(`[]`)必须保留——eval.yaml variant.allowedSkills
  *  优先于 strictBaseline 默认,漏挂会让 `--batch --config` 的 baseline 隔离配置静默失效。 */
 export declare function buildBatchVariantSpecs(entry: {
     name: string;

package/dist/eval-workflows/batch-evaluation-workflow.js

@@ -10,7 +10,7 @@ import { DEFAULT_JOBS_DIR } from '../eval-core/default-dirs.js';
  *  导致 #183 角色误绑各存一份)。
  *  两个 variant 的 allowedSkills 都从 eval.yaml variants[].allowedSkills 取:treatment 按 skill
  *  名 entry.name 查、baseline 按保留名 `baseline` 查,挂到对应 spec 上由 prepareEvaluationRun
- *  统一绑定。baseline 的显式声明(白名单或 `[]`)必须保留——eval.yaml variant.allowedSkills
+ *  统一绑定。baseline 的显式声明(`[]`)必须保留——eval.yaml variant.allowedSkills
  *  优先于 strictBaseline 默认,漏挂会让 `--batch --config` 的 baseline 隔离配置静默失效。 */
 export function buildBatchVariantSpecs(entry, variantAllowedSkills) {
     const baselineAllowed = variantAllowedSkills?.baseline;
@@ -98,7 +98,6 @@ export function buildBatchEvaluationReport({ batchRunId, skillDir, skillEntries,
         timeoutMs,
         noCache,
         dryRun: false,
-        blind: false,
         project,
         owner,
         tags,

package/dist/eval-workflows/evaluation-pipeline/report-finalize.d.ts

@@ -9,20 +9,16 @@
  *   - 可选 gapReports: 文本信号(markers / hedging)始终适用,与 tool trace 无关;
  *     带上 testSetHash 水印(spec §7.1 强制要求)
  *
- * 最后一步 `applyBlindMode` 是 blind 模式下的字段脱敏,放在所有 analysis 之后,
- * 避免脱敏后字段被分析逻辑读取。
- *
  * 仅被 orchestrator 调用;独立拆出主要为让 orchestrator 的 try-finally 主干
  * 看起来纯粹是「执行→收尾」时序。
  */
 import type { Artifact, Report, Sample, VariantResult } from '../../types/index.js';
 type EvaluationResults = Record<string, Record<string, VariantResult>>;
-export declare function finalizeEvaluationReport({ report, results, artifacts, variantNames, blind, samplesPath, samplesSourceFiles, samples, }: {
+export declare function finalizeEvaluationReport({ report, results, artifacts, variantNames, samplesPath, samplesSourceFiles, samples, }: {
     report: Report;
     results: EvaluationResults;
     artifacts: Artifact[];
     variantNames: string[];
-    blind: boolean;
     samplesPath: string;
     /** 目录模式下,bundle 内所有源文件;单文件模式下 [samplesPath]。computeTestSetHash 用。 */
     samplesSourceFiles?: string[];

package/dist/eval-workflows/evaluation-pipeline/report-finalize.js

@@ -9,18 +9,14 @@
  *   - 可选 gapReports: 文本信号(markers / hedging)始终适用,与 tool trace 无关;
  *     带上 testSetHash 水印(spec §7.1 强制要求)
  *
- * 最后一步 `applyBlindMode` 是 blind 模式下的字段脱敏,放在所有 analysis 之后,
- * 避免脱敏后字段被分析逻辑读取。
- *
  * 仅被 orchestrator 调用;独立拆出主要为让 orchestrator 的 try-finally 主干
  * 看起来纯粹是「执行→收尾」时序。
  */
 import { analyzeResults } from '../../analysis/report-diagnostics.js';
 import { computeReportCoverage } from '../../analysis/coverage-analyzer.js';
 import { computeReportGapRates } from '../../analysis/gap-analyzer.js';
-import { applyBlindMode } from '../../eval-core/evaluation-reporting.js';
 import { computeTestSetHash } from './test-set-hash.js';
-export function finalizeEvaluationReport({ report, results, artifacts, variantNames, blind, samplesPath, samplesSourceFiles, samples, }) {
+export function finalizeEvaluationReport({ report, results, artifacts, variantNames, samplesPath, samplesSourceFiles, samples, }) {
     // pass samples so analyzeResults can populate analysis.sampleQuality
     // (capability/difficulty/construct/provenance coverage aggregate). Without
     // samples, analysis.sampleQuality is omitted (老报告读取仍可工作).
@@ -49,8 +45,5 @@ export function finalizeEvaluationReport({ report, results, artifacts, variantNa
         }
         report.analysis.gapReports = gapReports;
     }
-    if (blind) {
-        applyBlindMode(report, variantNames, `${variantNames.join(',')}:${samplesPath}`);
-    }
     return report;
 }