oh-my-knowledge 0.40.0 → 0.41.0

package/dist/eval-workflows/evaluation-pipeline/run-state.d.ts

@@ -22,7 +22,7 @@ export interface EvaluationRunState {
     runningJob: EvaluationJob;
     resolvedJobStore: JobStore | null;
 }
-export declare function initializeEvaluationRunState({ samplesPath, skillDir, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, concurrency, timeoutMs, noCache, blind, project, owner, tags, runId, jobStore, persistJob, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }: {
+export declare function initializeEvaluationRunState({ samplesPath, skillDir, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, concurrency, timeoutMs, noCache, project, owner, tags, runId, jobStore, persistJob, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }: {
     samplesPath: string;
     skillDir: string;
     artifacts: Artifact[];
@@ -34,7 +34,6 @@ export declare function initializeEvaluationRunState({ samplesPath, skillDir, ar
     concurrency: number;
     timeoutMs?: number;
     noCache: boolean;
-    blind: boolean;
     project?: string;
     owner?: string;
     tags?: string[];

package/dist/eval-workflows/evaluation-pipeline/run-state.js

@@ -14,7 +14,7 @@
 import { buildEvaluationRequest, createFailedJob, createEvaluationRun, createQueuedJob, createSucceededJob, finalizeEvaluationRun, markJobRunning, failEvaluationRun, } from '../../eval-core/evaluation-job.js';
 import { createFileJobStore } from '../../server/job-store.js';
 import { DEFAULT_JOBS_DIR } from '../../eval-core/default-dirs.js';
-export async function initializeEvaluationRunState({ samplesPath, skillDir, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, concurrency, timeoutMs, noCache, blind, project, owner, tags, runId, jobStore, persistJob, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }) {
+export async function initializeEvaluationRunState({ samplesPath, skillDir, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, concurrency, timeoutMs, noCache, project, owner, tags, runId, jobStore, persistJob, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }) {
     const effectiveJudges = judgeModels && judgeModels.length > 0
         ? judgeModels
         : [{ executor: judgeExecutorName, model: judgeModel }];
@@ -29,7 +29,6 @@ export async function initializeEvaluationRunState({ samplesPath, skillDir, arti
         timeoutMs,
         noCache,
         dryRun: false,
-        blind,
         project,
         owner,
         tags,

package/dist/eval-workflows/evaluation-pipeline.d.ts

@@ -47,7 +47,6 @@ export interface EvaluationPipelineOptions {
     project?: string;
     owner?: string;
     tags?: string[];
-    blind?: boolean;
     concurrency?: number;
     timeoutMs?: number;
     noCache?: boolean;
@@ -90,7 +89,7 @@ export interface EvaluationPipelineOptions {
     noDiagnostic?: boolean;
 }
 type VariantResult = import('../types/index.js').VariantResult;
-export declare function executeEvaluationPipeline({ samplesPath, samplesBaseDir, samplesSourceFiles, skillDir, samples, tasks, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, executor, judgeExecutor, outputDir, project, owner, tags, blind, concurrency, timeoutMs, noCache, jobStore, persistJob, onProgress, skipConnectivity, verbose, retry, existingResults, requires: _requires, layeredStats, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, runId, lang, effort, noDiagnostic, }: EvaluationPipelineOptions): Promise<{
+export declare function executeEvaluationPipeline({ samplesPath, samplesBaseDir, samplesSourceFiles, skillDir, samples, tasks, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, executor, judgeExecutor, outputDir, project, owner, tags, concurrency, timeoutMs, noCache, jobStore, persistJob, onProgress, skipConnectivity, verbose, retry, existingResults, requires: _requires, layeredStats, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, runId, lang, effort, noDiagnostic, }: EvaluationPipelineOptions): Promise<{
     report: Report;
     filePath: string | null;
 }>;

package/dist/eval-workflows/evaluation-pipeline.js

@@ -30,7 +30,7 @@ import { emitIsolationWarnings, emitPowerWarnings } from './evaluation-pipeline/
 // 兼容 re-export:测试与 run-evaluation.ts 动态 import 仍打 evaluation-pipeline.js
 export { buildPowerWarnings, buildIsolationWarnings } from './evaluation-pipeline/preflight-warnings.js';
 export { _computeTestSetHashForTest } from './evaluation-pipeline/test-set-hash.js';
-export async function executeEvaluationPipeline({ samplesPath, samplesBaseDir, samplesSourceFiles, skillDir, samples, tasks, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, executor, judgeExecutor, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, blind = false, concurrency = 1, timeoutMs, noCache = false, jobStore = null, persistJob = true, onProgress = null, skipConnectivity = false, verbose = false, retry = 0, existingResults, 
+export async function executeEvaluationPipeline({ samplesPath, samplesBaseDir, samplesSourceFiles, skillDir, samples, tasks, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, executor, judgeExecutor, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, concurrency = 1, timeoutMs, noCache = false, jobStore = null, persistJob = true, onProgress = null, skipConnectivity = false, verbose = false, retry = 0, existingResults, 
 // requires 现在由 runEvaluation 上游传给 doctor 处理; eval-pipeline 不再用
 // 但保留接口字段,避免破 programmatic API (类型层面接收, 内部忽略)
 requires: _requires, layeredStats = false, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias = true, budget, strictBaseline, runId, lang = 'zh', effort, noDiagnostic, }) {
@@ -47,7 +47,6 @@ requires: _requires, layeredStats = false, repeat, batch, judgeRepeat, judgeMode
         concurrency,
         timeoutMs,
         noCache,
-        blind,
         project,
         owner,
         tags,
@@ -153,7 +152,6 @@ requires: _requires, layeredStats = false, repeat, batch, judgeRepeat, judgeMode
             results,
             artifacts,
             variantNames,
-            blind,
             samplesPath,
             samplesSourceFiles,
             samples,

package/dist/eval-workflows/run-evaluation.d.ts

@@ -46,7 +46,7 @@ interface CommonEvaluationOptions {
     bootstrap?: boolean;
     /** --bootstrap-samples N. Default 1000. */
     bootstrapSamples?: number;
-    /** length-debias toggle. Default true (judge prompt v3-cot-length).
+    /** length-debias toggle. Default true (length-debias instruction on).
      *  CLI passes false when --no-debias-length is set. */
     lengthDebias?: boolean;
     /** hard budget caps. */
@@ -64,7 +64,6 @@ export interface RunEvaluationOptions extends CommonEvaluationOptions {
     skillDir: string;
     variantSpecs?: VariantSpec[];
     dryRun?: boolean;
-    blind?: boolean;
     retry?: number;
     resume?: string;
     /** Explicit persisted run id. Used by batch workflows that need stable child ids. */
@@ -118,7 +117,7 @@ export interface DryRunReport extends DryRunBase {
     samplesPath: string;
     tasks: DryRunTask[];
 }
-export declare function runEvaluation({ samplesPath, skillDir, variantSpecs, model, outputDir, project, owner, tags, noJudge, dryRun, blind, concurrency, timeoutMs, noCache, executorName, jobStore, persistJob, onProgress, skipConnectivity, skipDoctor, lang, mcpConfig, verbose, retry, resume, runId, layeredStats, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, effort, noDiagnostic, }: RunEvaluationOptions): Promise<{
+export declare function runEvaluation({ samplesPath, skillDir, variantSpecs, model, outputDir, project, owner, tags, noJudge, dryRun, concurrency, timeoutMs, noCache, executorName, jobStore, persistJob, onProgress, skipConnectivity, skipDoctor, lang, mcpConfig, verbose, retry, resume, runId, layeredStats, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, effort, noDiagnostic, }: RunEvaluationOptions): Promise<{
     report: Report | DryRunReport;
     filePath: string | null;
 }>;

package/dist/eval-workflows/run-evaluation.js

@@ -8,7 +8,7 @@ import { buildDryRunTaskReport, prepareEvaluationRun, } from './evaluation-prepa
 import { executeEvaluationPipeline } from './evaluation-pipeline.js';
 import { findSaturationPoint } from '../analysis/saturation.js';
 import { bootstrapMeanCI, DEFAULT_BOOTSTRAP_ALPHA, DEFAULT_BOOTSTRAP_SAMPLES } from '../eval-core/bootstrap.js';
-export async function runEvaluation({ samplesPath, skillDir, variantSpecs = [], model = DEFAULT_MODEL, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, noJudge = false, dryRun = false, blind = false, concurrency = 1, timeoutMs, noCache = false, executorName = 'claude', jobStore = null, persistJob = true, onProgress = null, skipConnectivity = false, skipDoctor = false, lang = 'zh', mcpConfig, verbose = false, retry = 0, resume, runId, layeredStats = false, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, effort, noDiagnostic, }) {
+export async function runEvaluation({ samplesPath, skillDir, variantSpecs = [], model = DEFAULT_MODEL, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, noJudge = false, dryRun = false, concurrency = 1, timeoutMs, noCache = false, executorName = 'claude', jobStore = null, persistJob = true, onProgress = null, skipConnectivity = false, skipDoctor = false, lang = 'zh', mcpConfig, verbose = false, retry = 0, resume, runId, layeredStats = false, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, effort, noDiagnostic, }) {
     // Unified judgeModels → derive single-judge fields for downstream pipeline / grading
     // (which still operate on string `judgeModel` + `judgeExecutorName` fields per call).
     const effectiveJudgeModels = judgeModels && judgeModels.length > 0
@@ -152,7 +152,6 @@ export async function runEvaluation({ samplesPath, skillDir, variantSpecs = [],
         project,
         owner,
         tags,
-        blind,
         concurrency,
         timeoutMs,
         noCache,

package/dist/executors/claude-cli.js

@@ -2,20 +2,19 @@ import { extractAgentTrace, isClaudeSdkResultMessage } from './claude-sdk-trace.
 import { buildExecEnv, DEFAULT_TIMEOUT_MS, errorMessage, interruptedExecResult, MAX_BUFFER, spawnWithSigintPropagation, timeoutExecResult, } from './shared.js';
 import { materializeForCliConfigDir } from '../eval-core/mocks-runtime.js';
 // claude CLI 用 `--disable-slash-commands` (文档:"Disable all skills") +
-// `--disallowedTools Skill` 实现与 SDK 等价的完全隔离。但 CLI 没有 partial whitelist
-// 的 flag,所以 [name1, ...] 必须 throw。
+// `--disallowedTools Skill` 实现与 SDK 等价的完全隔离。非空 skill 白名单不再支持
+// (它无法真正隔离,见 claude-sdk.ts buildSdkIsolationOptions),所以 [name1, ...] 必须 throw。
 //
 //   undefined           → 不传任何 flag(原行为,全发现)
 //   []                  → --disable-slash-commands + --disallowedTools Skill
 //                         (main session skill discovery + subagent Skill 工具调用都堵)
-//   [...] (length > 0)  → throw,提示用 claude-sdk executor 走精准白名单
+//   [...] (length > 0)  → throw,非空白名单已移除
 function applySkillIsolationToCliArgs(args, allowedSkills) {
     if (allowedSkills === undefined)
         return;
     if (allowedSkills.length > 0) {
-        throw new Error(`claude-cli executor 不支持 partial skill 白名单(allowedSkills=${JSON.stringify(allowedSkills)})。\n`
-            + `  仅支持 [](映射为 --disable-slash-commands + --disallowedTools Skill)或 undefined(默认)。\n`
-            + `  精确白名单请改用 --executor claude-sdk(SDK skills option pass-through)。`);
+        throw new Error(`skill 白名单(allowedSkills=${JSON.stringify(allowedSkills)})不再支持:非空白名单无法真正隔离。\n`
+            + `  仅支持 [](映射为 --disable-slash-commands + --disallowedTools Skill,全封死)或 undefined(不隔离)。`);
     }
     // 完全隔离:双堵 main session skill 发现 + subagent Skill 工具
     args.push('--disable-slash-commands', '--disallowedTools', 'Skill');

package/dist/executors/claude-sdk.d.ts

@@ -3,8 +3,11 @@ import type { ExecResult, ExecutorInput } from '../types/index.js';
  * Map ExecutorInput.allowedSkills to SDK query options for skill isolation.
  *   undefined → {} (SDK default: full ~/.claude/skills/ discovery)
  *   []        → { skills: [], disallowedTools: ['Skill'] } (main session + subagent 双堵)
- *   [...]     → { skills: [...] } (main session whitelist; subagent 走独立 channel,
- *               白名单场景 v1 不强制 subagent 跟随)
+ *   [...]     → throw(非空 skill 白名单不再支持:它从不能真隔离 —— 主会话 skill 发现虽被
+ *               `skills:[...]` 收窄,但子代理 Skill 工具与 cwd 文件系统两条 channel 封不住,
+ *               会产出看着干净、实则被白名单外 skill 污染的报告。隔离只留两档:undefined
+ *               (不隔离)与 [](全封死),与 claude-cli / codex-cli 一致;多 skill 组合实验
+ *               请控制评测环境而非靠白名单。)
  *
  * Exported for unit tests to lock the option-shape contract.
  */

package/dist/executors/claude-sdk.js

@@ -9,8 +9,11 @@ let sdkQuery = null;
  * Map ExecutorInput.allowedSkills to SDK query options for skill isolation.
  *   undefined → {} (SDK default: full ~/.claude/skills/ discovery)
  *   []        → { skills: [], disallowedTools: ['Skill'] } (main session + subagent 双堵)
- *   [...]     → { skills: [...] } (main session whitelist; subagent 走独立 channel,
- *               白名单场景 v1 不强制 subagent 跟随)
+ *   [...]     → throw(非空 skill 白名单不再支持:它从不能真隔离 —— 主会话 skill 发现虽被
+ *               `skills:[...]` 收窄,但子代理 Skill 工具与 cwd 文件系统两条 channel 封不住,
+ *               会产出看着干净、实则被白名单外 skill 污染的报告。隔离只留两档:undefined
+ *               (不隔离)与 [](全封死),与 claude-cli / codex-cli 一致;多 skill 组合实验
+ *               请控制评测环境而非靠白名单。)
  *
  * Exported for unit tests to lock the option-shape contract.
  */
@@ -19,7 +22,8 @@ export function buildSdkIsolationOptions(allowedSkills) {
         return {};
     if (allowedSkills.length === 0)
         return { skills: allowedSkills, disallowedTools: ['Skill'] };
-    return { skills: allowedSkills };
+    throw new Error(`skill 白名单(allowedSkills=${JSON.stringify(allowedSkills)})不再支持:非空白名单无法真正隔离`
+        + `(子代理 Skill 工具 + cwd 文件系统两条 channel 封不住)。仅支持 [](全封死)或 undefined(不隔离)。`);
 }
 async function getSdkQuery() {
     if (!sdkQuery) {
@@ -29,6 +33,12 @@ async function getSdkQuery() {
     return sdkQuery;
 }
 export async function claudeSdkExecutor({ model, system, prompt, cwd, skillDir, timeoutMs = DEFAULT_TIMEOUT_MS, verbose = false, allowedSkills, mocks, mocksBaseDir, mocksStrict, lean, effort }) {
+    // 隔离选项在 timer / try 之前解析:非空 allowedSkills(不再支持的 skill 白名单)必须在这里
+    // fail-fast 抛错,而不是被下面的 catch 吞成 ok:false 的 ExecResult(那会把配置错误伪装成
+    // 每个 sample 执行失败、产出全失败报告,与 claude-cli / codex 的硬抛口径不一致)。lean 仍
+    // 覆盖为硬堵,但 buildSdkIsolationOptions 先跑一遍,故 lean 也无法绕过非空校验。
+    const baseIsolationOpts = buildSdkIsolationOptions(allowedSkills);
+    const isolationOpts = lean ? { skills: [], disallowedTools: ['*'] } : baseIsolationOpts;
     const start = Date.now();
     const abortController = new AbortController();
     const timer = setTimeout(() => abortController.abort(), timeoutMs);
@@ -45,11 +55,6 @@ export async function claudeSdkExecutor({ model, system, prompt, cwd, skillDir,
     const mockStatsOf = () => hookHandle ? { ...hookHandle.stats } : undefined;
     try {
         const query = await getSdkQuery();
-        // lean 模式:纯文本生成路径,不需要工具循环 / skill 发现,
-        // 用 disallowedTools:['*'] + skills:[] 直接堵住,优先级高于 isolationOpts。
-        const isolationOpts = lean
-            ? { skills: [], disallowedTools: ['*'] }
-            : buildSdkIsolationOptions(allowedSkills);
         // effort:lean 强制 'low'(生成路径不需要思考),否则透传调用方传入。
         // SDK 暴露 EffortLevel = 'low' | 'medium' | 'high' | 'xhigh' | 'max',直接对应。
         const effectiveEffort = lean ? 'low' : effort;

package/dist/executors/codex-cli.js

@@ -11,14 +11,13 @@ import { buildExecEnv, DEFAULT_TIMEOUT_MS, errorMessage, interruptedExecResult,
 //   undefined         → 不传 -C(原行为,看 cwd 里有什么 codex 自己决定)
 //   []                → 必须提供 cwd 非空(否则 throw),caller 应传一个
 //                       isolated 空目录(如 ~/.oh-my-knowledge/state/isolated-cwd/)
-//   [...] (length>0)  → throw,codex CLI 没有 partial 白名单 flag
+//   [...] (length>0)  → throw,非空白名单已移除(无法真正隔离)
 export function isolateCodexCwd(allowedSkills, cwd, executorName = 'codex-cli') {
     if (allowedSkills === undefined)
         return;
     if (allowedSkills.length > 0) {
-        throw new Error(`${executorName} executor 不支持 partial skill 白名单(allowedSkills=${JSON.stringify(allowedSkills)})。\n`
-            + `  仅支持 [](强制 cwd 隔离,需提供 cwd 非空)或 undefined(默认)。\n`
-            + `  codex CLI 无 partial 白名单 flag,请改用其他 executor 或显式 cwd 隔离。`);
+        throw new Error(`skill 白名单(allowedSkills=${JSON.stringify(allowedSkills)})不再支持:非空白名单无法真正隔离。\n`
+            + `  仅支持 [](强制 cwd 隔离,需提供 cwd 非空,全封死)或 undefined(不隔离)。`);
     }
     // allowedSkills === [] 时必须有 cwd(channel 3 cwd 隔离是 codex 唯一 channel)
     if (!cwd) {

package/dist/executors/shared.d.ts

@@ -5,6 +5,8 @@ export declare const DEFAULT_MODEL = "sonnet";
 export declare const JUDGE_MODEL = "haiku";
 export declare const DEFAULT_TIMEOUT_MS = 600000;
 export declare const MAX_BUFFER: number;
+export type ExecutorVendor = 'anthropic' | 'openai' | 'google' | 'unknown';
+export declare function executorVendor(executor: string): ExecutorVendor;
 export interface TokenUsage {
     input_tokens?: number;
     output_tokens?: number;

package/dist/executors/shared.js

@@ -9,6 +9,21 @@ export const DEFAULT_MODEL = 'sonnet';
 export const JUDGE_MODEL = 'haiku';
 export const DEFAULT_TIMEOUT_MS = 600_000;
 export const MAX_BUFFER = 10 * 1024 * 1024;
+// 把 executor 名归到模型厂商家族,用于评委独立性判断(同厂商评委 = 自我偏好敞口)。
+// codex 是 OpenAI 的代码模型,故 codex* 归 OpenAI。自定义 script executor 无法归类 → unknown
+// (调用方据此跳过判断,不误报)。
+const EXECUTOR_VENDOR = {
+    claude: 'anthropic',
+    'claude-sdk': 'anthropic',
+    'anthropic-api': 'anthropic',
+    codex: 'openai',
+    'codex-sdk': 'openai',
+    'openai-api': 'openai',
+    gemini: 'google',
+};
+export function executorVendor(executor) {
+    return EXECUTOR_VENDOR[executor] ?? 'unknown';
+}
 export function asErrorLike(err) {
     return typeof err === 'object' && err !== null ? err : {};
 }

package/dist/grading/assertions.js

@@ -1,6 +1,7 @@
 import { resolve } from 'node:path';
 import _Ajv from 'ajv';
 import { ASSERTION_LAYER } from './layered-scores.js';
+import { buildSemanticSimilarityPrompt, SEMANTIC_SIMILARITY_SYSTEM, buildRagJudgePrompt } from '../shared/llm-prompts/judge-prompts.js';
 const Ajv = _Ajv.default ?? _Ajv;
 const ajv = new Ajv();
 const CUSTOM_ASSERTION_TIMEOUT_MS = 30_000;
@@ -332,24 +333,10 @@ export async function runAsyncAssertions(output, assertions, { executor, judgeMo
         let message = '';
         if (assertion.type === 'semantic_similarity') {
             const reference = assertion.reference || '';
-            const judgePrompt = [
-                '请判断以下两段文本的语义相似度。',
-                '',
-                '## 参考文本',
-                reference,
-                '',
-                '## 待评估文本',
-                output,
-                '',
-                '请返回 JSON（不要包含 markdown 代码块标记）：',
-                '{"score": <1-5的整数>, "reason": "<简短理由>"}',
-                '',
-                '评分：1=完全无关, 2=略有关联, 3=部分相似, 4=大致相同, 5=高度一致',
-            ].join('\n');
             const result = await executor({
                 model: judgeModel,
-                system: '你是语义相似度评审员。只返回 JSON，不要其他内容。',
-                prompt: judgePrompt,
+                system: SEMANTIC_SIMILARITY_SYSTEM,
+                prompt: buildSemanticSimilarityPrompt(reference, output),
             });
             asyncCostUSD += result.costUSD || 0;
             if (result.costReportedByExecutor === false)
@@ -423,40 +410,6 @@ export async function runAsyncAssertions(output, assertions, { executor, judgeMo
         ...(anyCostUnreported && { judgeCostReportedByExecutor: false }),
     };
 }
-// ===========================================================================
-// RAG-specific judge metrics
-// ===========================================================================
-//
-// Three metrics, all running through the LLM judge:
-//
-//  - faithfulness:      does the output's content stay grounded in the
-//                       reference context? Anti-hallucination check.
-//  - answer_relevancy:  does the output directly answer the user's question?
-//                       Catches verbose dodges and topic drift.
-//  - context_recall:    are the key facts from the gold context actually
-//                       used in the output? Catches retrieved-but-ignored
-//                       context (a common RAG bug).
-//
-// Implementation notes:
-//
-//  1. Each prompt is a SINGLE-CALL judge (1-5 score) rather than the multi-step
-//     statement-decomposition that RAGAS uses. This is honest tradeoff: simpler,
-//     faster, less rigorous than RAGAS but consistent with omk's other LLM-judge
-//     assertions. Users who need RAGAS-grade decomposition can drop down to a
-//     custom assertion.
-//  2. The prompt includes the SAME length-debias paragraph as the main judge
-//     prompt (v3-cot-length) — output verbosity is not a quality signal here
-//     either. This is the "auto-inherit length-debias" claim from the  plan.
-//  3. Reference resolution:
-//       faithfulness:    sample.context  (or assertion.reference override)
-//       context_recall:  assertion.reference (or sample.context fallback)
-//       answer_relevancy: sample.prompt — no reference needed
-//  4. Threshold defaults to 3 (same as semantic_similarity). User can override.
-const RAG_LENGTH_DEBIAS = [
-    '## 重要:长度不是质量信号',
-    '评分时聚焦内容实质,不要因输出更长就给更高分。',
-    '简洁正确的回答与冗长正确的回答应得相同分数。',
-].join('\n');
 async function runRagJudge(assertion, output, sample, executor, judgeModel) {
     const threshold = assertion.threshold ?? 3;
     let prompt;
@@ -466,55 +419,10 @@ async function runRagJudge(assertion, output, sample, executor, judgeModel) {
         if (!context) {
             return { passed: false, message: 'faithfulness: 缺少 sample.context 或 assertion.reference', costUSD: 0 };
         }
-        system = '你是 RAG 评审员,专注判断输出是否被参考 context 支持。只返回 JSON。';
-        prompt = [
-            '请判断"待评估输出"中的事实性陈述是否被"参考 context"支持。',
-            '',
-            '## 参考 context',
-            context,
-            '',
-            '## 待评估输出',
-            output,
-            '',
-            RAG_LENGTH_DEBIAS,
-            '',
-            '## 评分流程',
-            '1. 列出待评估输出中所有事实性陈述',
-            '2. 逐条判断是否能在 context 中找到支持',
-            '3. 给出 1-5 分:',
-            '   5 = 全部陈述都有 context 支持,无编造',
-            '   4 = 多数有支持,有 1-2 处不重要的编造',
-            '   3 = 一半有支持',
-            '   2 = 多数无支持',
-            '   1 = 完全编造或与 context 矛盾',
-            '',
-            '请返回 JSON(不要 markdown 代码块):',
-            '{"score": <1-5的整数>, "reason": "<简短理由>"}',
-        ].join('\n');
+        ({ system, prompt } = buildRagJudgePrompt('faithfulness', { output, context }));
     }
     else if (assertion.type === 'answer_relevancy') {
-        system = '你是答题切题度评审员。只返回 JSON。';
-        prompt = [
-            '请判断"AI 输出"是否直接、切题地回答了"用户问题"。',
-            '',
-            '## 用户问题',
-            sample.prompt,
-            '',
-            '## AI 输出',
-            output,
-            '',
-            RAG_LENGTH_DEBIAS,
-            '',
-            '## 评分',
-            '5 = 完整切题回答,无冗余无遗漏',
-            '4 = 切题但有少量冗余或小遗漏',
-            '3 = 部分切题,部分跑题或避而不答',
-            '2 = 大部分跑题',
-            '1 = 完全跑题或拒答',
-            '',
-            '请返回 JSON(不要 markdown 代码块):',
-            '{"score": <1-5的整数>, "reason": "<简短理由>"}',
-        ].join('\n');
+        ({ system, prompt } = buildRagJudgePrompt('answer_relevancy', { output, question: sample.prompt }));
     }
     else {
         // context_recall
@@ -522,31 +430,7 @@ async function runRagJudge(assertion, output, sample, executor, judgeModel) {
         if (!reference) {
             return { passed: false, message: 'context_recall: 缺少 assertion.reference 或 sample.context', costUSD: 0 };
         }
-        system = '你是 context 覆盖率评审员。只返回 JSON。';
-        prompt = [
-            '请判断"参考 gold"中的关键事实在"AI 输出"中被覆盖的程度。',
-            '',
-            '## 参考 gold',
-            reference,
-            '',
-            '## AI 输出',
-            output,
-            '',
-            RAG_LENGTH_DEBIAS,
-            '',
-            '## 评分流程',
-            '1. 列出参考中的关键事实(忽略修饰性内容)',
-            '2. 检查每条是否在输出中被提及/使用',
-            '3. 给出 1-5 分:',
-            '   5 = 全部关键事实被覆盖',
-            '   4 = 大部分覆盖,缺 1-2 条次要事实',
-            '   3 = 一半覆盖',
-            '   2 = 仅覆盖少量',
-            '   1 = 完全未覆盖',
-            '',
-            '请返回 JSON(不要 markdown 代码块):',
-            '{"score": <1-5的整数>, "reason": "<简短理由>"}',
-        ].join('\n');
+        ({ system, prompt } = buildRagJudgePrompt('context_recall', { output, reference }));
     }
     const result = await executor({ model: judgeModel, system, prompt });
     const reported = result.costReportedByExecutor === false ? { costReportedByExecutor: false } : {};

package/dist/grading/gold-cli.js

@@ -93,7 +93,7 @@ export function formatGoldCompare(result, gold) {
             lines.push(`  报告缺失:         ${result.missing.join(', ')}`);
         return lines.join('\n');
     }
-    lines.push(`  Krippendorff α:   ${fmt(a.alpha)}   (主指标，序数加权)`);
+    lines.push(`  Krippendorff α:   ${fmt(a.alpha)}   (主指标，区间加权)`);
     lines.push(`  α 95% CI:         [${fmt(a.alphaCI.low)}, ${fmt(a.alphaCI.high)}]`);
     lines.push(`  加权 κ:           ${fmt(a.weightedKappa)}   (副指标)`);
     lines.push(`  Pearson r:        ${fmt(a.pearson)}   (仅查 rank order)`);

package/dist/grading/human-gold.d.ts

@@ -15,9 +15,11 @@
  *
  * Three metrics are exported:
  *
- *  - **Krippendorff's α (interval)** — primary. Distribution-free, supports
- *    ordinal/interval scales naturally, doesn't assume coders are exchangeable
- *    (good fit when one "coder" is a model and the other a human annotator).
+ *  - **Krippendorff's α (interval weights)** — primary. Distribution-free,
+ *    doesn't assume coders are exchangeable (good fit when one "coder" is a model
+ *    and the other a human annotator). Uses interval distance δ²=(c−k)² — a
+ *    defensible choice for 1-5 Likert; an ordinal-distance variant would change α
+ *    (BREAKING-COMPARABILITY) and is not implemented here.
  *  - **Quadratic-weighted Cohen's κ** — secondary. Familiar to many readers,
  *    useful as a sanity check. Reports lower than α when marginals diverge.
  *  - **Pearson r** — tertiary. Captures rank-order agreement only; doesn't

package/dist/grading/human-gold.js

@@ -15,9 +15,11 @@
  *
  * Three metrics are exported:
  *
- *  - **Krippendorff's α (interval)** — primary. Distribution-free, supports
- *    ordinal/interval scales naturally, doesn't assume coders are exchangeable
- *    (good fit when one "coder" is a model and the other a human annotator).
+ *  - **Krippendorff's α (interval weights)** — primary. Distribution-free,
+ *    doesn't assume coders are exchangeable (good fit when one "coder" is a model
+ *    and the other a human annotator). Uses interval distance δ²=(c−k)² — a
+ *    defensible choice for 1-5 Likert; an ordinal-distance variant would change α
+ *    (BREAKING-COMPARABILITY) and is not implemented here.
  *  - **Quadratic-weighted Cohen's κ** — secondary. Familiar to many readers,
  *    useful as a sanity check. Reports lower than α when marginals diverge.
  *  - **Pearson r** — tertiary. Captures rank-order agreement only; doesn't

package/dist/grading/index.d.ts

@@ -45,10 +45,10 @@ interface GradeOptions {
      */
     judgeRepeat?: number;
     /**
-     * v0.21 length-debias toggle. Defaults to true — judge prompt includes the
-     * "length is not a quality signal" instruction, prompt template version is
-     * v3-cot-length. Set false (via `--no-debias-length`) to revert to the
-     * legacy v2-cot prompt for reproducing historical reports.
+     * length-debias toggle. Defaults to true — judge prompt includes the
+     * "length is not a quality signal" instruction. Set false (via `--no-debias-length`)
+     * to drop it (the debias-off prompt variant) for reproducing older no-length-debias
+     * reports. The presentation/tone neutrality instruction is always on regardless.
      */
     lengthDebias?: boolean;
 }

package/dist/grading/judge.d.ts

@@ -1,13 +1,5 @@
 import type { DimensionResult, ExecutorFn, JudgeAgreement, JudgeConfig, ToolCallInfo, TurnInfo } from '../types/index.js';
-export declare function buildJudgePrompt(prompt: string, rubric: string, output: string, traceSummary: string | null, lengthDebias?: boolean): string;
-/**
- * Stable hash of the judge prompt template. Saved into ReportMeta.judgePromptHash so
- * downstream readers can detect "the judge prompt changed between these two reports".
- *
- * `lengthDebias` defaults to true (v0.21+ default). Pass false when running under
- * `--no-debias-length` so the hash matches historical v2-cot reports.
- */
-export declare function getJudgePromptHash(lengthDebias?: boolean): string;
+export { buildJudgePrompt, getJudgePromptHash } from '../shared/llm-prompts/judge-prompts.js';
 interface LlmJudgeOptions {
     output: string;
     rubric: string;
@@ -17,9 +9,10 @@ interface LlmJudgeOptions {
     traceSummary?: string | null;
     /**
      * When true (default), the judge prompt includes an explicit
-     * "length is not a quality signal" instruction. Pass false to fall back to
-     * the legacy v2-cot prompt — only useful for reproducing pre-v0.21 reports
-     * or running A/B comparisons with alternate length-debias settings.
+     * "length is not a quality signal" instruction. Pass false to drop it (the
+     * debias-off prompt variant) — only useful for reproducing older no-length-debias
+     * reports or running A/B comparisons with alternate length-debias settings.
+     * (The presentation/tone neutrality instruction is always on, independent of this.)
      */
     lengthDebias?: boolean;
 }
@@ -60,7 +53,7 @@ export declare function judgeId(config: JudgeConfig): string;
 export declare function computeJudgeAgreement(judgeScores: number[][]): JudgeAgreement;
 /**
  * Judge a single (output, rubric) pair with N judge models in parallel. Each judge
- * may use a different executor (e.g. claude:opus + openai:gpt-4o + gemini:pro). Each
+ * may use a different executor (e.g. claude:opus + openai-api:gpt-4o + gemini:pro). Each
  * judge can also be repeated `judgeRepeat` times — final per-judge score is its mean.
  *
  * Returns: aggregate DimensionResult (score = mean across judges; this is the "consensus"
@@ -72,4 +65,3 @@ export declare function computeJudgeAgreement(judgeScores: number[][]): JudgeAgr
  * provide the raw ensemble so downstream can recompute.
  */
 export declare function llmJudgeEnsemble(options: LlmJudgeOptions, judges: JudgeConfig[], executorByName: (name: string) => ExecutorFn, judgeRepeat?: number): Promise<DimensionResult>;
-export {};