oh-my-knowledge 0.40.0 → 0.42.0

package/dist/shared/llm-prompts/judge-prompts.js

@@ -0,0 +1,205 @@
+import { createHash } from 'node:crypto';
+import { LENGTH_DEBIAS_INSTRUCTION, PRESENTATION_NEUTRALITY_INSTRUCTION, RAG_LENGTH_DEBIAS, RAG_PRESENTATION_NEUTRALITY, } from './debias-instructions.js';
+// 评分类 prompt 单一来源 —— 直接决定分数的 LLM 评委 prompt（rubric 主评委 + RAG + 语义相似度）。
+// 这些是测量学不变量:文本字节决定可比性,改动须配合 prompt-registry 的冻结 hash bump
+// (BREAKING-COMPARABILITY)。执行器调用 / JSON 解析逻辑留在 grading/judge.ts 与 grading/assertions.ts。
+// ===========================================================================
+// Rubric 主评委 prompt
+// ===========================================================================
+/**
+ * Judge prompt template version.
+ *
+ * The version string encodes WHICH debias / context features the judge sees, so reports
+ * tagged with the same hash are score-comparable; a mismatched hash means "we changed how
+ * we ask the judge to think" and the reports should not be compared blind. Bump it (and the
+ * frozen hashes in `test/shared/prompt-registry-freeze.test.ts`) whenever the template's bytes
+ * change — that change is BREAKING-COMPARABILITY.
+ *
+ * Naming: a single main version (`v5`) + a `-feature` suffix per debias/context capability.
+ * The only asymmetry between the two strings is the `-len` suffix, gated by the
+ * `--no-debias-length` toggle; every other feature is always-on and appears in both.
+ */
+// 版本演进(内嵌"评委看到什么 / 被要求忽略什么"的语义):
+//   v2-cot               : 仅看 output + rubric + 工具名分布(早期 buildTraceSummary)
+//   v3-cot-length        : 加 length-debias 指令
+//   v3-cot-toolargs(off) / v4-cot-len-args(on)
+//                        : 加 tool input 预览(让评委识别 wrapper-style skill 在 Bash 命令里的
+//                          真实语义调用,如 `Bash: mcporter --tool X`,否则误判"只调了 Bash")
+//   v5-cot-toolargs-fmt(off) / v5-cot-toolargs-fmt-len(on)
+//                        : 加排版 / 语气中性化指令(始终开启,不给开关)。研究表明 LLM 评委除了
+//                          偏向更长的回答,还隐性偏向排版精致(标题 / 列表 / 加粗)与语气自信 / 自我
+//                          表扬(谄媚 / 权威偏置)的回答;显式指令要求评委只对照评分标准核内容。
+//                          同时借此把命名统一成单一主序号(v5)+ feature 后缀,`-len` 仍是开关那条。
+const JUDGE_PROMPT_VERSION_DEBIAS_OFF = 'v5-cot-toolargs-fmt';
+const JUDGE_PROMPT_VERSION_DEBIAS_ON = 'v5-cot-toolargs-fmt-len';
+export const JUDGE_SYSTEM_PROMPT = '你是一个严格的 AI 输出质量评审员。先逐条对照评分标准做推理，再给最终分数。只返回 JSON，不要其他内容。';
+export function buildJudgePrompt(prompt, rubric, output, traceSummary, lengthDebias = true) {
+    const version = lengthDebias ? JUDGE_PROMPT_VERSION_DEBIAS_ON : JUDGE_PROMPT_VERSION_DEBIAS_OFF;
+    const traceSection = traceSummary
+        ? ['', '## Agent 执行过程', traceSummary, '', '请同时考虑执行过程的合理性（工具选择、步骤效率、错误恢复）。']
+        : [];
+    // 排版 / 语气中性化始终开启(不受 --no-debias-length 影响);length-debias 仍受开关控。
+    const neutralitySection = ['', PRESENTATION_NEUTRALITY_INSTRUCTION];
+    const debiasSection = lengthDebias ? ['', LENGTH_DEBIAS_INSTRUCTION] : [];
+    return [
+        `请对以下 AI 输出进行质量评分（template ${version}）。`,
+        '',
+        '## 原始任务',
+        prompt,
+        '',
+        '## 评分标准',
+        rubric,
+        '',
+        '## AI 输出',
+        output,
+        ...traceSection,
+        ...neutralitySection,
+        ...debiasSection,
+        '',
+        '## 评分流程',
+        '1. 逐条对照评分标准，先做推理（reasoning）：列出 AI 输出哪些点对应哪条标准，哪些缺失，哪些有歧义。',
+        '2. 基于推理给出最终分数（1-5 的整数）和简短理由。',
+        '',
+        '请返回 JSON（不要包含 markdown 代码块标记）：',
+        '{"reasoning": "<对照标准的逐条推理>", "score": <1-5的整数>, "reason": "<最终结论的简短理由>"}',
+        '',
+        '评分标准：1=完全不达标, 2=部分涉及, 3=基本达标, 4=较好, 5=优秀',
+    ].join('\n');
+}
+/**
+ * Stable hash of the judge prompt template. Saved into ReportMeta.judgePromptHash so
+ * downstream readers can detect "the judge prompt changed between these two reports".
+ *
+ * `lengthDebias` defaults to true. Pass false (via `--no-debias-length`) to drop the
+ * length-debias instruction; that produces the debias-off prompt variant, whose hash
+ * differs from the default so the two are never compared blind.
+ */
+export function getJudgePromptHash(lengthDebias = true) {
+    const version = lengthDebias ? JUDGE_PROMPT_VERSION_DEBIAS_ON : JUDGE_PROMPT_VERSION_DEBIAS_OFF;
+    // Hash the template-shaping function source + the version tag together. We hash a
+    // deterministic stringified form of the template (with placeholder inputs) so any
+    // structural edit shows up.
+    const sample = buildJudgePrompt('<P>', '<R>', '<O>', '<T>', lengthDebias);
+    return createHash('sha256').update(version + '\n' + sample).digest('hex').slice(0, 12);
+}
+// ===========================================================================
+// 语义相似度评委 prompt（semantic_similarity 断言）
+// ===========================================================================
+export const SEMANTIC_SIMILARITY_SYSTEM = '你是语义相似度评审员。只返回 JSON，不要其他内容。';
+export function buildSemanticSimilarityPrompt(reference, output) {
+    return [
+        '请判断以下两段文本的语义相似度。',
+        '',
+        '## 参考文本',
+        reference,
+        '',
+        '## 待评估文本',
+        output,
+        '',
+        '请返回 JSON（不要包含 markdown 代码块标记）：',
+        '{"score": <1-5的整数>, "reason": "<简短理由>"}',
+        '',
+        '评分：1=完全无关, 2=略有关联, 3=部分相似, 4=大致相同, 5=高度一致',
+    ].join('\n');
+}
+/** RAG 评委 system + 用户 prompt。缺字段的早退 / 解析逻辑留在 assertions.ts。 */
+export function buildRagJudgePrompt(type, fields) {
+    if (type === 'faithfulness') {
+        return {
+            system: '你是 RAG 评审员,专注判断输出是否被参考 context 支持。只返回 JSON。',
+            prompt: [
+                '请判断"待评估输出"中的事实性陈述是否被"参考 context"支持。',
+                '',
+                '## 参考 context',
+                fields.context ?? '',
+                '',
+                '## 待评估输出',
+                fields.output,
+                '',
+                RAG_LENGTH_DEBIAS,
+                RAG_PRESENTATION_NEUTRALITY,
+                '',
+                '## 评分流程',
+                '1. 列出待评估输出中所有事实性陈述',
+                '2. 逐条判断是否能在 context 中找到支持',
+                '3. 给出 1-5 分:',
+                '   5 = 全部陈述都有 context 支持,无编造',
+                '   4 = 多数有支持,有 1-2 处不重要的编造',
+                '   3 = 一半有支持',
+                '   2 = 多数无支持',
+                '   1 = 完全编造或与 context 矛盾',
+                '',
+                '请返回 JSON(不要 markdown 代码块):',
+                '{"score": <1-5的整数>, "reason": "<简短理由>"}',
+            ].join('\n'),
+        };
+    }
+    if (type === 'answer_relevancy') {
+        return {
+            system: '你是答题切题度评审员。只返回 JSON。',
+            prompt: [
+                '请判断"AI 输出"是否直接、切题地回答了"用户问题"。',
+                '',
+                '## 用户问题',
+                fields.question ?? '',
+                '',
+                '## AI 输出',
+                fields.output,
+                '',
+                RAG_LENGTH_DEBIAS,
+                RAG_PRESENTATION_NEUTRALITY,
+                '',
+                '## 评分',
+                '5 = 完整切题回答,无冗余无遗漏',
+                '4 = 切题但有少量冗余或小遗漏',
+                '3 = 部分切题,部分跑题或避而不答',
+                '2 = 大部分跑题',
+                '1 = 完全跑题或拒答',
+                '',
+                '请返回 JSON(不要 markdown 代码块):',
+                '{"score": <1-5的整数>, "reason": "<简短理由>"}',
+            ].join('\n'),
+        };
+    }
+    // context_recall
+    return {
+        system: '你是 context 覆盖率评审员。只返回 JSON。',
+        prompt: [
+            '请判断"参考 gold"中的关键事实在"AI 输出"中被覆盖的程度。',
+            '',
+            '## 参考 gold',
+            fields.reference ?? '',
+            '',
+            '## AI 输出',
+            fields.output,
+            '',
+            RAG_LENGTH_DEBIAS,
+            RAG_PRESENTATION_NEUTRALITY,
+            '',
+            '## 评分流程',
+            '1. 列出参考中的关键事实(忽略修饰性内容)',
+            '2. 检查每条是否在输出中被提及/使用',
+            '3. 给出 1-5 分:',
+            '   5 = 全部关键事实被覆盖',
+            '   4 = 大部分覆盖,缺 1-2 条次要事实',
+            '   3 = 一半覆盖',
+            '   2 = 仅覆盖少量',
+            '   1 = 完全未覆盖',
+            '',
+            '请返回 JSON(不要 markdown 代码块):',
+            '{"score": <1-5的整数>, "reason": "<简短理由>"}',
+        ].join('\n'),
+    };
+}
+/** 评分类断言 prompt 的 drift-detection hash(占位符输入,12 位,仅供 prompt-registry 冻结)。 */
+function hashPromptSample(label, system, prompt) {
+    return createHash('sha256').update(`${label}\n${system}\n${prompt}`).digest('hex').slice(0, 12);
+}
+export function getSemanticPromptHash() {
+    return hashPromptSample('semantic_similarity', SEMANTIC_SIMILARITY_SYSTEM, buildSemanticSimilarityPrompt('<R>', '<O>'));
+}
+export function getRagJudgePromptHash(type) {
+    const fields = { output: '<O>', context: '<C>', question: '<Q>', reference: '<R>' };
+    const { system, prompt } = buildRagJudgePrompt(type, fields);
+    return hashPromptSample(`rag:${type}`, system, prompt);
+}

package/dist/shared/llm-prompts/registry.d.ts

@@ -0,0 +1,27 @@
+/**
+ * omk 所有 LLM prompt 的单一编目 —— 发现入口 + 测量学冻结入口。
+ *
+ * 解决「prompt 散在多个目录、难管理」与「只有 2 个 prompt 被冻结」两个问题:
+ *   - 每条登记 prompt 的位置(module)+ 用途,一处可查全。
+ *   - measurementInvariant 为 true 的(直接决定分数的评委 prompt)带 getHash,
+ *     由 test/shared/prompt-registry-freeze.test.ts 统一冻结,文本漂移即 CI 红。
+ *
+ * 注意:hash 形状按各自历史口径,不强行统一 ——
+ *   - rubric 评委:12 位(getJudgePromptHash,持久化进 report.meta.judgePromptHash 的同口径);
+ *   - RAG / semantic:12 位(getRagJudgePromptHash / getSemanticPromptHash,仅供冻结 drift 检测);
+ *   - observe 复盘:64 位(readPromptDocument 读 .md 全文 sha256)。
+ * 用限定名 `promptId`(非裸 `kind`,见 terminology-spec §5.4)。
+ */
+export interface PromptRegistryEntry {
+    /** 稳定标识(冻结表的 key)。 */
+    promptId: string;
+    /** 人类可读用途。 */
+    purpose: string;
+    /** 定义位置(发现用,相对仓库根的源文件路径)。 */
+    module: string;
+    /** 是否直接决定分数 → 须冻结防漂移。 */
+    measurementInvariant: boolean;
+    /** 仅 measurementInvariant 条目提供:当前 prompt 的 hash。 */
+    getHash?: () => string;
+}
+export declare const PROMPT_REGISTRY: PromptRegistryEntry[];

package/dist/shared/llm-prompts/registry.js

@@ -0,0 +1,69 @@
+import { getJudgePromptHash, getSemanticPromptHash, getRagJudgePromptHash } from './judge-prompts.js';
+import { readPromptDocument } from './index.js';
+import { PROMPTS_DIR, SOFT_STANDARD_PROMPT_ID, SOFT_STANDARD_PROMPT_VERSION } from '../../observability/soft-standards/constants.js';
+export const PROMPT_REGISTRY = [
+    // —— 测量学不变量:直接决定分数的评委 prompt,统一冻结 ——
+    {
+        promptId: 'rubric-judge-debias-on',
+        purpose: 'rubric 主评委(length-debias 开,默认)',
+        module: 'src/shared/llm-prompts/judge-prompts.ts',
+        measurementInvariant: true,
+        getHash: () => getJudgePromptHash(true),
+    },
+    {
+        promptId: 'rubric-judge-debias-off',
+        purpose: 'rubric 主评委(--no-debias-length,length-debias 关)',
+        module: 'src/shared/llm-prompts/judge-prompts.ts',
+        measurementInvariant: true,
+        getHash: () => getJudgePromptHash(false),
+    },
+    {
+        promptId: 'semantic-similarity',
+        purpose: 'semantic_similarity 断言评委',
+        module: 'src/shared/llm-prompts/judge-prompts.ts',
+        measurementInvariant: true,
+        getHash: () => getSemanticPromptHash(),
+    },
+    {
+        promptId: 'rag-faithfulness',
+        purpose: 'RAG faithfulness(输出是否被 context 支持)',
+        module: 'src/shared/llm-prompts/judge-prompts.ts',
+        measurementInvariant: true,
+        getHash: () => getRagJudgePromptHash('faithfulness'),
+    },
+    {
+        promptId: 'rag-answer-relevancy',
+        purpose: 'RAG answer_relevancy(是否切题)',
+        module: 'src/shared/llm-prompts/judge-prompts.ts',
+        measurementInvariant: true,
+        getHash: () => getRagJudgePromptHash('answer_relevancy'),
+    },
+    {
+        promptId: 'rag-context-recall',
+        purpose: 'RAG context_recall(关键事实覆盖率)',
+        module: 'src/shared/llm-prompts/judge-prompts.ts',
+        measurementInvariant: true,
+        getHash: () => getRagJudgePromptHash('context_recall'),
+    },
+    {
+        promptId: 'observe-llm-enhanced-review',
+        purpose: 'observe 运行期软标准抽取 / LLM 增强复盘',
+        module: 'src/observability/prompts/llm-enhanced-review.prompt.md',
+        measurementInvariant: true,
+        getHash: () => readPromptDocument({
+            dir: PROMPTS_DIR,
+            fileName: `${SOFT_STANDARD_PROMPT_ID}.prompt.md`,
+            id: SOFT_STANDARD_PROMPT_ID,
+            version: SOFT_STANDARD_PROMPT_VERSION,
+        }).hash,
+    },
+    // —— 非评分类:不决定 composite / verdict / assertion 分数,不冻结,仅登记供发现 ——
+    { promptId: 'failure-diagnostic', purpose: '失败诊断(root cause / 修复建议)', module: 'src/grading/diagnostic.ts', measurementInvariant: false },
+    { promptId: 'failure-clusterer', purpose: '失败案例聚类', module: 'src/analysis/failure-clusterer.ts', measurementInvariant: false },
+    { promptId: 'hedging-classifier', purpose: 'hedging 判定(喂 gap-signal,非评分)', module: 'src/analysis/hedging-classifier.ts', measurementInvariant: false },
+    { promptId: 'sample-generator', purpose: '用例生成(skill / trace → samples)', module: 'src/authoring/generator.ts', measurementInvariant: false },
+    { promptId: 'sample-fixer', purpose: '坏用例修复', module: 'src/authoring/sample-fixer.ts', measurementInvariant: false },
+    { promptId: 'skill-improve', purpose: 'skill 迭代改进(evolve)', module: 'src/authoring/evolver.ts', measurementInvariant: false },
+    { promptId: 'doctor-fixer', purpose: 'doctor 健康项修复向导', module: 'src/doctor/fixer.ts', measurementInvariant: false },
+    { promptId: 'skill-health', purpose: 'skill 健康检查打分', module: 'src/shared/llm-prompts/skill-health.ts', measurementInvariant: false },
+];

package/dist/types/eval.d.ts

@@ -211,7 +211,6 @@ export interface EvalConfig {
     timeoutMs?: number;
     noCache?: boolean;
     noJudge?: boolean;
-    blind?: boolean;
     mcpConfig?: string;
     variants: EvalConfigVariant[];
     /** hard budget caps. When any limit is hit during a run, remaining
@@ -221,6 +220,9 @@ export interface EvalConfig {
     budget?: EvalBudget;
     /** --repeat N. Multi-run variance analysis. */
     repeat?: number;
+    /** --holdout-ratio R (0 < R < 1). Hold out a deterministic sample slice and
+     *  report train vs holdout composite as a generalization / overfitting signal. */
+    holdoutRatio?: number;
     /** --judge-repeat N. Each (sample × dimension) judged N times for self-consistency stddev. */
     judgeRepeat?: number;
     /** --bootstrap. Distribution-free CI per variant + pairwise diff. */
@@ -229,7 +231,7 @@ export interface EvalConfig {
     bootstrapSamples?: number;
     /** --gold-dir. After-run automatic comparison against a human-anchor dataset. */
     goldDir?: string;
-    /** --no-debias-length flips this to false. Default true (judge prompt v3-cot-length). */
+    /** --no-debias-length flips this to false. Default true (length-debias instruction on). */
     lengthDebias?: boolean;
     /** --no-strict-baseline flips this to false. Default true (baseline-kind allowedSkills=[]). */
     strictBaseline?: boolean;
@@ -256,9 +258,12 @@ export interface EvaluationRequest {
     timeoutMs?: number;
     noCache: boolean;
     dryRun: boolean;
-    blind: boolean;
     /** --repeat N; 1 表示单次跑,> 1 走 runMultiple 做 variance 分析 */
     repeat?: number;
+    /** --holdout-ratio R; 0 / 缺省表示不切分(默认)。> 0 时 report-finalize 在结果上
+     *  post-hoc 切出 train / holdout 子集算综合分(`report.analysis.holdout`),供 verdict
+     *  的过拟合门控读取。see src/eval-core/holdout.ts */
+    holdoutRatio?: number;
     /** --batch; default absent/false. True means skill-batch mode. */
     batch?: boolean;
     /** --judge-repeat N; 每条 sample × dimension 用 LLM judge 跑 N 次, 输出 stddev. 默认 1 (单次). */
@@ -266,8 +271,9 @@ export interface EvaluationRequest {
     /** Unified judge config — always non-empty.
      *  - length === 1: single judge (degenerate ensemble of size 1).
      *  - length >= 2: multi-judge ensemble. Each (sample × dimension) is scored by every judge,
-     *    inter-judge agreement (Pearson + mean absolute difference) reported as the hard
-     *    rebuttal to "judge same-model bias".
+     *    inter-judge agreement (Pearson + mean absolute difference) reported as a rebuttal to
+     *    "judge same-model bias" — but only when judges span vendors; a single-vendor ensemble's
+     *    high agreement reflects shared bias, not independence (flagged by `single_vendor_ensemble`).
      *  When `noJudge: true` the entry is preserved for audit but no judge call actually runs. */
     judgeModels: JudgeConfig[];
     /** --bootstrap; true 时 aggregateReport 加跑 bootstrap mean/diff CI, 写入 VariantSummary.
@@ -275,9 +281,10 @@ export interface EvaluationRequest {
     bootstrap?: boolean;
     /** --bootstrap-samples N; bootstrap 重采样次数, 默认 1000. > 10000 时 stderr 警告. */
     bootstrapSamples?: number;
-    /** length-debias toggle. Default true (judge prompt v3-cot-length).
-     *  CLI flag --no-debias-length flips to false (legacy v2-cot prompt). The active
-     *  value is reflected in ReportMeta.judgePromptHash and ReportMeta.debiasMode. */
+    /** length-debias toggle. Default true — judge prompt carries the length-debias
+     *  instruction. CLI flag --no-debias-length flips to false (drops that instruction,
+     *  the debias-off prompt variant). The active value is reflected in
+     *  ReportMeta.judgePromptHash and ReportMeta.debiasMode. */
     lengthDebias?: boolean;
     /** hard budget caps. See EvalBudget. */
     budget?: EvalBudget;

package/dist/types/judge.d.ts

@@ -18,7 +18,7 @@ export interface JudgeRuntimeEntry {
 }
 /** Per-judge ensemble entry: which judge gave what score (mean over judge-repeat if N>1). */
 export interface EnsembleJudgeResult {
-    /** "executor:model" identifier — e.g. "claude:opus" or "openai:gpt-4o". */
+    /** "executor:model" identifier — e.g. "claude:opus" or "openai-api:gpt-4o". */
     judge: string;
     /** Mean score from this judge over judge-repeat calls (or single score if repeat=1). */
     score: number;

package/dist/types/report.d.ts

@@ -317,7 +317,7 @@ export interface ReportMeta {
     humanAgreement?: ReportHumanAgreement;
     variantConfigs?: VariantConfig[];
     /** Skill isolation 快照(per-variant)。
-     *  key = variant name;value = allowedSkills(undefined → null,SDK 默认全发现 / [] → 完全隔离 / [...] → 白名单)。
+     *  key = variant name;value = allowedSkills(undefined → null,SDK 默认全发现 / [] → 完全隔离;非空白名单已移除)。
      *  跨报告对比 verdict / Δ 时,isolation 状态不一致会被 stderr warn 标"不可比"。
      *  字段缺失意味着报告产自  之前(默认全发现,construct validity 不保证)。 */
     skillIsolation?: Record<string, string[] | null>;
@@ -325,8 +325,6 @@ export interface ReportMeta {
     run?: EvaluationRun;
     job?: EvaluationJob;
     gitInfo?: GitInfo | null;
-    blind?: boolean;
-    blindMap?: Record<string, string>;
     layeredStats?: boolean;
     /** Evolve 合并报告的原始 skill 归属。variants 会被 relabel 为 round-0/round-1,
      *  Studio skill 索引用该字段把报告归回 skill 卡片。 */
@@ -482,6 +480,34 @@ export interface AnalysisResult {
      *  (capability / difficulty / construct / provenance); persisted on report
      *  for studio to surface coverage gaps. See docs/specs/sample-design-spec.md. */
     sampleQuality?: SampleQualityAggregate;
+    /** Opt-in train/holdout generalization breakdown (`omk eval --holdout-ratio`).
+     *  Absent on default runs; present only when a holdout ratio was requested. */
+    holdout?: HoldoutBreakdown;
+}
+/** Train vs holdout composite breakdown for `omk eval --holdout-ratio`.
+ *  Computed post-hoc from `report.results` by `computeHoldoutBreakdown`
+ *  (`src/eval-core/holdout.ts`), sharing the same testSetHash watermark as
+ *  gapReports (gap-spec §7.1). A large train − holdout composite gap is the
+ *  sample-set-overfitting signal the verdict's overfitting gate reads. */
+export interface HoldoutBreakdown {
+    /** Held-out fraction requested via --holdout-ratio. */
+    ratio: number;
+    /** true when either side fell below the minimum subset size → scored full-set,
+     *  no usable split. `perVariant` is empty and the verdict gate stays inert. */
+    disabled?: boolean;
+    /** Per-variant train vs holdout composite (1-5 scale). `*Count` is the authored
+     *  split size; `*Scorable` is how many of those actually produced a composite (> 0)
+     *  — they diverge under partial errors, and the overfitting gate trusts `*Scorable`. */
+    perVariant: Record<string, {
+        trainScore: number;
+        holdoutScore: number;
+        trainCount: number;
+        holdoutCount: number;
+        trainScorable: number;
+        holdoutScorable: number;
+    }>;
+    testSetPath?: string | null;
+    testSetHash?: string | null;
 }
 /** Aggregated sample design coverage stats. Built by
  *  `buildSampleQualityAggregate(samples)` from `Sample.capability` /
@@ -504,6 +530,27 @@ export interface SampleQualityAggregate {
     sampleCountWithDifficulty: number;
     sampleCountWithConstruct: number;
     sampleCountWithProvenance: number;
+    /** Relative-balance / skew of the sample set (derived from the distributions
+     *  above). Flags over-representation — "70% of samples are easy" — without an
+     *  external denominator. Diagnostic only; never feeds grading / judge / verdict. */
+    representativeness?: Representativeness;
+}
+/** Distribution skew over what the sample set declares. Pure relative balance —
+ *  there is no authored "expected" capability list to measure absolute coverage
+ *  against (capabilities are free-form strings), so this reports concentration
+ *  (dominant bucket share, 0-1) and the dominant label per dimension. */
+export interface Representativeness {
+    /** Distinct capabilities declared across the set. */
+    capabilityCount: number;
+    /** Dominant capability's share of all capability tags (0-1); 0 when none declared. */
+    capabilityConcentration: number;
+    dominantCapability?: string;
+    /** Dominant difficulty bucket's share of samples that declared a difficulty (0-1). */
+    difficultyConcentration: number;
+    dominantDifficulty?: 'easy' | 'medium' | 'hard';
+    /** Dominant construct's share of samples that declared a construct (0-1). */
+    constructConcentration: number;
+    dominantConstruct?: string;
 }
 export interface HedgingVerdict {
     isUncertainty: boolean;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "oh-my-knowledge",
-  "version": "0.40.0",
+  "version": "0.42.0",
   "packageManager": "yarn@4.16.0",
   "description": "Evaluation framework for LLM knowledge inputs — prompts, RAG corpora, skills, agent workflows. Fix the model, vary the artifact. Built-in statistical rigor: bootstrap CI, Krippendorff α, length-debias, saturation curves.",
   "type": "module",

package/dist/grading/debias-validate.d.ts

@@ -1,83 +0,0 @@
-/**
- * Measure how much the judge's scores shift when the length-debias instruction
- * is toggled.
- *
- * What it actually measures
- * -------------------------
- * Given a finished report, this command re-judges every (sample, variant) pair
- * using the OPPOSITE length-debias setting from the original run. It then
- * compares the two score distributions with a bootstrap CI on the mean
- * difference.
- *
- * If the original report ran with debias-on (v3-cot-length), we re-judge with
- * v2-cot (legacy). If the original ran with debias-off, we re-judge with
- * v3-cot-length. Significant difference → the prompt change moves scores → the
- * judge is sensitive to the length-debias instruction. That's *consistent with*
- * length bias being present, but it doesn't prove it directly — a perfectly
- * length-neutral judge could in principle also be sensitive to the wording for
- * other reasons. We label the verdict accordingly.
- *
- * Cost
- * ----
- * Re-judging is a full second pass over all (sample, variant) cells. Cost
- * doubles vs the original run. The CLI surfaces this so users running on
- * large/expensive evaluations can opt in deliberately.
- */
-import type { ExecutorFn, Report, Sample } from '../types/index.js';
-import { type BootstrapDiffCI } from '../eval-core/bootstrap.js';
-export interface DebiasValidateInput {
-    report: Report;
-    samples: Sample[];
-    judgeExecutor: ExecutorFn;
-    judgeModel: string;
-    /** Variant to validate. Defaults to first variant. */
-    variant?: string;
-    /** Bootstrap iterations for the diff CI. Default 1000. */
-    bootstrapSamples?: number;
-    seed?: number;
-    /** Progress hook. */
-    onProgress?: (info: {
-        sample_id: string;
-        completed: number;
-        total: number;
-    }) => void;
-}
-export interface DebiasValidateResult {
-    variant: string;
-    /** Original lengthDebias setting (true if debias-on at run time). */
-    originalLengthDebias: boolean;
-    /** Pairs of (originalScore, alternateScore) per sample. */
-    pairs: Array<{
-        sample_id: string;
-        originalScore: number;
-        alternateScore: number;
-    }>;
-    meanOriginal: number;
-    meanAlternate: number;
-    /** Mean of (alternate - original). Positive = alternate prompt scored higher. */
-    diffCI: BootstrapDiffCI;
-    /** Verdict in the {未检测, 弱, 中, 强} bucket plus an English shadow. */
-    verdict: {
-        zh: string;
-        en: string;
-        level: 'none' | 'weak' | 'medium' | 'strong';
-    };
-    /** Total cost burned re-judging. */
-    alternateJudgeCostUSD: number;
-    /** Sample_ids that the report had but lacked judge scores. */
-    unscored: string[];
-    /** Sample_ids in the samples file that are missing from the report. */
-    missing: string[];
-}
-/**
- * Re-judge every sample of `variant` in the given report with the OPPOSITE
- * lengthDebias setting and compute the bootstrap CI on the mean difference.
- *
- * The judge call uses the rubric from the samples file and the output stored
- * in the report — we do NOT re-execute the model. Only judging is repeated.
- *
- * Multi-dimensional samples currently use the rubric as fallback when there's
- * no top-level rubric. Per-dimension validation can be added later if needed.
- */
-export declare function validateLengthDebias(input: DebiasValidateInput): Promise<DebiasValidateResult>;
-export declare function formatDebiasValidate(result: DebiasValidateResult): string;