oh-my-knowledge 0.40.0 → 0.41.0

package/dist/shared/llm-prompts/registry.js

@@ -0,0 +1,69 @@
+import { getJudgePromptHash, getSemanticPromptHash, getRagJudgePromptHash } from './judge-prompts.js';
+import { readPromptDocument } from './index.js';
+import { PROMPTS_DIR, SOFT_STANDARD_PROMPT_ID, SOFT_STANDARD_PROMPT_VERSION } from '../../observability/soft-standards/constants.js';
+export const PROMPT_REGISTRY = [
+    // —— 测量学不变量:直接决定分数的评委 prompt,统一冻结 ——
+    {
+        promptId: 'rubric-judge-debias-on',
+        purpose: 'rubric 主评委(length-debias 开,默认)',
+        module: 'src/shared/llm-prompts/judge-prompts.ts',
+        measurementInvariant: true,
+        getHash: () => getJudgePromptHash(true),
+    },
+    {
+        promptId: 'rubric-judge-debias-off',
+        purpose: 'rubric 主评委(--no-debias-length,length-debias 关)',
+        module: 'src/shared/llm-prompts/judge-prompts.ts',
+        measurementInvariant: true,
+        getHash: () => getJudgePromptHash(false),
+    },
+    {
+        promptId: 'semantic-similarity',
+        purpose: 'semantic_similarity 断言评委',
+        module: 'src/shared/llm-prompts/judge-prompts.ts',
+        measurementInvariant: true,
+        getHash: () => getSemanticPromptHash(),
+    },
+    {
+        promptId: 'rag-faithfulness',
+        purpose: 'RAG faithfulness(输出是否被 context 支持)',
+        module: 'src/shared/llm-prompts/judge-prompts.ts',
+        measurementInvariant: true,
+        getHash: () => getRagJudgePromptHash('faithfulness'),
+    },
+    {
+        promptId: 'rag-answer-relevancy',
+        purpose: 'RAG answer_relevancy(是否切题)',
+        module: 'src/shared/llm-prompts/judge-prompts.ts',
+        measurementInvariant: true,
+        getHash: () => getRagJudgePromptHash('answer_relevancy'),
+    },
+    {
+        promptId: 'rag-context-recall',
+        purpose: 'RAG context_recall(关键事实覆盖率)',
+        module: 'src/shared/llm-prompts/judge-prompts.ts',
+        measurementInvariant: true,
+        getHash: () => getRagJudgePromptHash('context_recall'),
+    },
+    {
+        promptId: 'observe-llm-enhanced-review',
+        purpose: 'observe 运行期软标准抽取 / LLM 增强复盘',
+        module: 'src/observability/prompts/llm-enhanced-review.prompt.md',
+        measurementInvariant: true,
+        getHash: () => readPromptDocument({
+            dir: PROMPTS_DIR,
+            fileName: `${SOFT_STANDARD_PROMPT_ID}.prompt.md`,
+            id: SOFT_STANDARD_PROMPT_ID,
+            version: SOFT_STANDARD_PROMPT_VERSION,
+        }).hash,
+    },
+    // —— 非评分类:不决定 composite / verdict / assertion 分数,不冻结,仅登记供发现 ——
+    { promptId: 'failure-diagnostic', purpose: '失败诊断(root cause / 修复建议)', module: 'src/grading/diagnostic.ts', measurementInvariant: false },
+    { promptId: 'failure-clusterer', purpose: '失败案例聚类', module: 'src/analysis/failure-clusterer.ts', measurementInvariant: false },
+    { promptId: 'hedging-classifier', purpose: 'hedging 判定(喂 gap-signal,非评分)', module: 'src/analysis/hedging-classifier.ts', measurementInvariant: false },
+    { promptId: 'sample-generator', purpose: '用例生成(skill / trace → samples)', module: 'src/authoring/generator.ts', measurementInvariant: false },
+    { promptId: 'sample-fixer', purpose: '坏用例修复', module: 'src/authoring/sample-fixer.ts', measurementInvariant: false },
+    { promptId: 'skill-improve', purpose: 'skill 迭代改进(evolve)', module: 'src/authoring/evolver.ts', measurementInvariant: false },
+    { promptId: 'doctor-fixer', purpose: 'doctor 健康项修复向导', module: 'src/doctor/fixer.ts', measurementInvariant: false },
+    { promptId: 'skill-health', purpose: 'skill 健康检查打分', module: 'src/shared/llm-prompts/skill-health.ts', measurementInvariant: false },
+];

package/dist/types/eval.d.ts

@@ -211,7 +211,6 @@ export interface EvalConfig {
     timeoutMs?: number;
     noCache?: boolean;
     noJudge?: boolean;
-    blind?: boolean;
     mcpConfig?: string;
     variants: EvalConfigVariant[];
     /** hard budget caps. When any limit is hit during a run, remaining
@@ -229,7 +228,7 @@ export interface EvalConfig {
     bootstrapSamples?: number;
     /** --gold-dir. After-run automatic comparison against a human-anchor dataset. */
     goldDir?: string;
-    /** --no-debias-length flips this to false. Default true (judge prompt v3-cot-length). */
+    /** --no-debias-length flips this to false. Default true (length-debias instruction on). */
     lengthDebias?: boolean;
     /** --no-strict-baseline flips this to false. Default true (baseline-kind allowedSkills=[]). */
     strictBaseline?: boolean;
@@ -256,7 +255,6 @@ export interface EvaluationRequest {
     timeoutMs?: number;
     noCache: boolean;
     dryRun: boolean;
-    blind: boolean;
     /** --repeat N; 1 表示单次跑,> 1 走 runMultiple 做 variance 分析 */
     repeat?: number;
     /** --batch; default absent/false. True means skill-batch mode. */
@@ -266,8 +264,9 @@ export interface EvaluationRequest {
     /** Unified judge config — always non-empty.
      *  - length === 1: single judge (degenerate ensemble of size 1).
      *  - length >= 2: multi-judge ensemble. Each (sample × dimension) is scored by every judge,
-     *    inter-judge agreement (Pearson + mean absolute difference) reported as the hard
-     *    rebuttal to "judge same-model bias".
+     *    inter-judge agreement (Pearson + mean absolute difference) reported as a rebuttal to
+     *    "judge same-model bias" — but only when judges span vendors; a single-vendor ensemble's
+     *    high agreement reflects shared bias, not independence (flagged by `single_vendor_ensemble`).
      *  When `noJudge: true` the entry is preserved for audit but no judge call actually runs. */
     judgeModels: JudgeConfig[];
     /** --bootstrap; true 时 aggregateReport 加跑 bootstrap mean/diff CI, 写入 VariantSummary.
@@ -275,9 +274,10 @@ export interface EvaluationRequest {
     bootstrap?: boolean;
     /** --bootstrap-samples N; bootstrap 重采样次数, 默认 1000. > 10000 时 stderr 警告. */
     bootstrapSamples?: number;
-    /** length-debias toggle. Default true (judge prompt v3-cot-length).
-     *  CLI flag --no-debias-length flips to false (legacy v2-cot prompt). The active
-     *  value is reflected in ReportMeta.judgePromptHash and ReportMeta.debiasMode. */
+    /** length-debias toggle. Default true — judge prompt carries the length-debias
+     *  instruction. CLI flag --no-debias-length flips to false (drops that instruction,
+     *  the debias-off prompt variant). The active value is reflected in
+     *  ReportMeta.judgePromptHash and ReportMeta.debiasMode. */
     lengthDebias?: boolean;
     /** hard budget caps. See EvalBudget. */
     budget?: EvalBudget;

package/dist/types/judge.d.ts

@@ -18,7 +18,7 @@ export interface JudgeRuntimeEntry {
 }
 /** Per-judge ensemble entry: which judge gave what score (mean over judge-repeat if N>1). */
 export interface EnsembleJudgeResult {
-    /** "executor:model" identifier — e.g. "claude:opus" or "openai:gpt-4o". */
+    /** "executor:model" identifier — e.g. "claude:opus" or "openai-api:gpt-4o". */
     judge: string;
     /** Mean score from this judge over judge-repeat calls (or single score if repeat=1). */
     score: number;

package/dist/types/report.d.ts

@@ -317,7 +317,7 @@ export interface ReportMeta {
     humanAgreement?: ReportHumanAgreement;
     variantConfigs?: VariantConfig[];
     /** Skill isolation 快照(per-variant)。
-     *  key = variant name;value = allowedSkills(undefined → null,SDK 默认全发现 / [] → 完全隔离 / [...] → 白名单)。
+     *  key = variant name;value = allowedSkills(undefined → null,SDK 默认全发现 / [] → 完全隔离;非空白名单已移除)。
      *  跨报告对比 verdict / Δ 时,isolation 状态不一致会被 stderr warn 标"不可比"。
      *  字段缺失意味着报告产自  之前(默认全发现,construct validity 不保证)。 */
     skillIsolation?: Record<string, string[] | null>;
@@ -325,8 +325,6 @@ export interface ReportMeta {
     run?: EvaluationRun;
     job?: EvaluationJob;
     gitInfo?: GitInfo | null;
-    blind?: boolean;
-    blindMap?: Record<string, string>;
     layeredStats?: boolean;
     /** Evolve 合并报告的原始 skill 归属。variants 会被 relabel 为 round-0/round-1,
      *  Studio skill 索引用该字段把报告归回 skill 卡片。 */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "oh-my-knowledge",
-  "version": "0.40.0",
+  "version": "0.41.0",
   "packageManager": "yarn@4.16.0",
   "description": "Evaluation framework for LLM knowledge inputs — prompts, RAG corpora, skills, agent workflows. Fix the model, vary the artifact. Built-in statistical rigor: bootstrap CI, Krippendorff α, length-debias, saturation curves.",
   "type": "module",

package/dist/grading/debias-validate.d.ts

@@ -1,83 +0,0 @@
-/**
- * Measure how much the judge's scores shift when the length-debias instruction
- * is toggled.
- *
- * What it actually measures
- * -------------------------
- * Given a finished report, this command re-judges every (sample, variant) pair
- * using the OPPOSITE length-debias setting from the original run. It then
- * compares the two score distributions with a bootstrap CI on the mean
- * difference.
- *
- * If the original report ran with debias-on (v3-cot-length), we re-judge with
- * v2-cot (legacy). If the original ran with debias-off, we re-judge with
- * v3-cot-length. Significant difference → the prompt change moves scores → the
- * judge is sensitive to the length-debias instruction. That's *consistent with*
- * length bias being present, but it doesn't prove it directly — a perfectly
- * length-neutral judge could in principle also be sensitive to the wording for
- * other reasons. We label the verdict accordingly.
- *
- * Cost
- * ----
- * Re-judging is a full second pass over all (sample, variant) cells. Cost
- * doubles vs the original run. The CLI surfaces this so users running on
- * large/expensive evaluations can opt in deliberately.
- */
-import type { ExecutorFn, Report, Sample } from '../types/index.js';
-import { type BootstrapDiffCI } from '../eval-core/bootstrap.js';
-export interface DebiasValidateInput {
-    report: Report;
-    samples: Sample[];
-    judgeExecutor: ExecutorFn;
-    judgeModel: string;
-    /** Variant to validate. Defaults to first variant. */
-    variant?: string;
-    /** Bootstrap iterations for the diff CI. Default 1000. */
-    bootstrapSamples?: number;
-    seed?: number;
-    /** Progress hook. */
-    onProgress?: (info: {
-        sample_id: string;
-        completed: number;
-        total: number;
-    }) => void;
-}
-export interface DebiasValidateResult {
-    variant: string;
-    /** Original lengthDebias setting (true if debias-on at run time). */
-    originalLengthDebias: boolean;
-    /** Pairs of (originalScore, alternateScore) per sample. */
-    pairs: Array<{
-        sample_id: string;
-        originalScore: number;
-        alternateScore: number;
-    }>;
-    meanOriginal: number;
-    meanAlternate: number;
-    /** Mean of (alternate - original). Positive = alternate prompt scored higher. */
-    diffCI: BootstrapDiffCI;
-    /** Verdict in the {未检测, 弱, 中, 强} bucket plus an English shadow. */
-    verdict: {
-        zh: string;
-        en: string;
-        level: 'none' | 'weak' | 'medium' | 'strong';
-    };
-    /** Total cost burned re-judging. */
-    alternateJudgeCostUSD: number;
-    /** Sample_ids that the report had but lacked judge scores. */
-    unscored: string[];
-    /** Sample_ids in the samples file that are missing from the report. */
-    missing: string[];
-}
-/**
- * Re-judge every sample of `variant` in the given report with the OPPOSITE
- * lengthDebias setting and compute the bootstrap CI on the mean difference.
- *
- * The judge call uses the rubric from the samples file and the output stored
- * in the report — we do NOT re-execute the model. Only judging is repeated.
- *
- * Multi-dimensional samples currently use the rubric as fallback when there's
- * no top-level rubric. Per-dimension validation can be added later if needed.
- */
-export declare function validateLengthDebias(input: DebiasValidateInput): Promise<DebiasValidateResult>;
-export declare function formatDebiasValidate(result: DebiasValidateResult): string;

package/dist/grading/debias-validate.js

@@ -1,176 +0,0 @@
-/**
- * Measure how much the judge's scores shift when the length-debias instruction
- * is toggled.
- *
- * What it actually measures
- * -------------------------
- * Given a finished report, this command re-judges every (sample, variant) pair
- * using the OPPOSITE length-debias setting from the original run. It then
- * compares the two score distributions with a bootstrap CI on the mean
- * difference.
- *
- * If the original report ran with debias-on (v3-cot-length), we re-judge with
- * v2-cot (legacy). If the original ran with debias-off, we re-judge with
- * v3-cot-length. Significant difference → the prompt change moves scores → the
- * judge is sensitive to the length-debias instruction. That's *consistent with*
- * length bias being present, but it doesn't prove it directly — a perfectly
- * length-neutral judge could in principle also be sensitive to the wording for
- * other reasons. We label the verdict accordingly.
- *
- * Cost
- * ----
- * Re-judging is a full second pass over all (sample, variant) cells. Cost
- * doubles vs the original run. The CLI surfaces this so users running on
- * large/expensive evaluations can opt in deliberately.
- */
-import { llmJudge } from './judge.js';
-import { bootstrapPairedDiffCI } from '../eval-core/bootstrap.js';
-/**
- * Map a bootstrap diff CI to a verdict bucket. The ranges are deliberately
- * conservative: we only label "strong" when the CI fully sits >= |0.5| away
- * from zero (about half a point on a 1-5 scale).
- */
-function classifyVerdict(diff) {
-    if (!diff.significant) {
-        return { zh: '未检测到显著差异', en: 'no significant shift', level: 'none' };
-    }
-    const mag = Math.min(Math.abs(diff.low), Math.abs(diff.high));
-    if (mag >= 0.5) {
-        return { zh: '强差异——prompt 改动对评分影响大', en: 'strong shift', level: 'strong' };
-    }
-    if (mag >= 0.2) {
-        return { zh: '中等差异——校正对结论有实质影响', en: 'medium shift', level: 'medium' };
-    }
-    return { zh: '弱差异——显著但幅度小', en: 'weak shift', level: 'weak' };
-}
-/**
- * Re-judge every sample of `variant` in the given report with the OPPOSITE
- * lengthDebias setting and compute the bootstrap CI on the mean difference.
- *
- * The judge call uses the rubric from the samples file and the output stored
- * in the report — we do NOT re-execute the model. Only judging is repeated.
- *
- * Multi-dimensional samples currently use the rubric as fallback when there's
- * no top-level rubric. Per-dimension validation can be added later if needed.
- */
-export async function validateLengthDebias(input) {
-    const { report, samples, judgeExecutor, judgeModel, bootstrapSamples = 1000, seed, onProgress } = input;
-    const variant = input.variant ?? report.meta.variants?.[0];
-    if (!variant)
-        throw new Error('report has no variants — nothing to validate');
-    // Detect original lengthDebias setting from meta. Default to true (v0.21+).
-    // Older reports without debiasMode set are treated as legacy (debias-off).
-    const debiasModeList = report.meta.debiasMode ?? [];
-    const originalLengthDebias = debiasModeList.includes('length');
-    const alternateLengthDebias = !originalLengthDebias;
-    const sampleById = new Map();
-    for (const s of samples)
-        sampleById.set(s.sample_id, s);
-    // Pre-pass: collect (sample_id, output, originalScore) tuples.
-    const tasks = [];
-    const unscored = [];
-    const missing = [];
-    for (const entry of report.results ?? []) {
-        const v = entry.variants?.[variant];
-        if (!v || !v.fullOutput)
-            continue;
-        const sample = sampleById.get(entry.sample_id);
-        if (!sample) {
-            missing.push(entry.sample_id);
-            continue;
-        }
-        if (typeof v.llmScore !== 'number' || v.llmScore <= 0) {
-            unscored.push(entry.sample_id);
-            continue;
-        }
-        tasks.push({ sample, output: v.fullOutput, originalScore: v.llmScore });
-    }
-    // Re-judge with the opposite debias setting. We use the simplest path:
-    // single-rubric judge. Multi-dim samples fall back to the explicit rubric
-    // string if available. Samples without a rubric are skipped — we have
-    // nothing to feed the judge.
-    const pairs = [];
-    let alternateJudgeCostUSD = 0;
-    let completed = 0;
-    for (const t of tasks) {
-        completed++;
-        onProgress?.({ sample_id: t.sample.sample_id, completed, total: tasks.length });
-        const rubric = t.sample.rubric
-            ?? (t.sample.dimensions ? Object.values(t.sample.dimensions).join('\n') : '');
-        if (!rubric)
-            continue;
-        const altResult = await llmJudge({
-            output: t.output,
-            rubric,
-            prompt: t.sample.prompt,
-            executor: judgeExecutor,
-            model: judgeModel,
-            lengthDebias: alternateLengthDebias,
-        });
-        if (altResult.judgeCostUSD)
-            alternateJudgeCostUSD += altResult.judgeCostUSD;
-        if (altResult.score > 0) {
-            pairs.push({
-                sample_id: t.sample.sample_id,
-                originalScore: t.originalScore,
-                alternateScore: altResult.score,
-            });
-        }
-    }
-    const meanOriginal = avg(pairs.map((p) => p.originalScore));
-    const meanAlternate = avg(pairs.map((p) => p.alternateScore));
-    // **配对** diff CI:每个 sample 同时有 original 与 alternate prompt 两个分数(同一回答、两个 judge prompt
-    // = 配对设计)。原先拆成两数组喂独立重采样,丢弃了配对、高估方差、CI 偏宽 —— 对一个**检测**长度偏置敏感性
-    // 的工具,保守方向恰好是错的(更难检出真实偏置)。改配对:重采样 sample 下标、按 (alternate − original) 算,
-    // 保留 within-sample 相关、收紧 CI,提升对偏置的检出力。diff = b − a = alternate − original(同原约定)。
-    const diffCI = bootstrapPairedDiffCI(pairs.map((p) => ({ a: p.originalScore, b: p.alternateScore })), 0.05, bootstrapSamples, seed);
-    const verdict = classifyVerdict(diffCI);
-    return {
-        variant,
-        originalLengthDebias,
-        pairs,
-        meanOriginal: Number(meanOriginal.toFixed(3)),
-        meanAlternate: Number(meanAlternate.toFixed(3)),
-        diffCI,
-        verdict,
-        alternateJudgeCostUSD: Number(alternateJudgeCostUSD.toFixed(6)),
-        unscored,
-        missing,
-    };
-}
-function avg(arr) {
-    if (arr.length === 0)
-        return 0;
-    return arr.reduce((s, x) => s + x, 0) / arr.length;
-}
-export function formatDebiasValidate(result) {
-    const lines = [];
-    const dirOrig = result.originalLengthDebias ? 'on (v3-cot-length)' : 'off (v2-cot)';
-    const dirAlt = result.originalLengthDebias ? 'off (v2-cot)' : 'on (v3-cot-length)';
-    lines.push(`\n  Length-debias 灵敏度验证 (variant: ${result.variant})\n`);
-    lines.push(`  原始 prompt:    ${dirOrig}`);
-    lines.push(`  对照 prompt:    ${dirAlt}`);
-    lines.push(`  用例数:         ${result.pairs.length}`);
-    if (result.pairs.length === 0) {
-        lines.push('  无可比对用例——检查报告是否含 fullOutput / rubric。');
-        return lines.join('\n');
-    }
-    lines.push(`  原均值:         ${result.meanOriginal.toFixed(3)}`);
-    lines.push(`  对照均值:       ${result.meanAlternate.toFixed(3)}`);
-    const ci = result.diffCI;
-    lines.push(`  差值 (alt-orig): ${ci.estimate >= 0 ? '+' : ''}${ci.estimate}`);
-    lines.push(`  95% CI:         [${ci.low}, ${ci.high}]   (${ci.significant ? '显著' : '不显著'})`);
-    lines.push('');
-    lines.push(`  结论:           ${result.verdict.zh}`);
-    lines.push('');
-    lines.push(`  注: 该结论反映"prompt 切换是否改变评分"。差异显著 = 评分对 length-debias 指令敏感,`);
-    lines.push(`      间接支持 length bias 存在;但 prompt 文本变化也可能因其他原因影响评分。`);
-    lines.push(`      重判 cost: ${result.alternateJudgeCostUSD.toFixed(6)} USD`);
-    if (result.missing.length) {
-        lines.push(`  缺用例 ID:      ${result.missing.length} 条`);
-    }
-    if (result.unscored.length) {
-        lines.push(`  无 LLM 分:      ${result.unscored.length} 条`);
-    }
-    return lines.join('\n');
-}