oh-my-knowledge 0.40.0 → 0.42.0

package/dist/eval-workflows/evaluation-pipeline/report-finalize.js

@@ -9,18 +9,15 @@
  *   - 可选 gapReports: 文本信号(markers / hedging)始终适用,与 tool trace 无关;
  *     带上 testSetHash 水印(spec §7.1 强制要求)
  *
- * 最后一步 `applyBlindMode` 是 blind 模式下的字段脱敏,放在所有 analysis 之后,
- * 避免脱敏后字段被分析逻辑读取。
- *
  * 仅被 orchestrator 调用;独立拆出主要为让 orchestrator 的 try-finally 主干
  * 看起来纯粹是「执行→收尾」时序。
  */
 import { analyzeResults } from '../../analysis/report-diagnostics.js';
 import { computeReportCoverage } from '../../analysis/coverage-analyzer.js';
 import { computeReportGapRates } from '../../analysis/gap-analyzer.js';
-import { applyBlindMode } from '../../eval-core/evaluation-reporting.js';
+import { computeHoldoutBreakdown } from '../../eval-core/holdout.js';
 import { computeTestSetHash } from './test-set-hash.js';
-export function finalizeEvaluationReport({ report, results, artifacts, variantNames, blind, samplesPath, samplesSourceFiles, samples, }) {
+export function finalizeEvaluationReport({ report, results, artifacts, variantNames, samplesPath, samplesSourceFiles, samples, }) {
     // pass samples so analyzeResults can populate analysis.sampleQuality
     // (capability/difficulty/construct/provenance coverage aggregate). Without
     // samples, analysis.sampleQuality is omitted (老报告读取仍可工作).
@@ -37,9 +34,13 @@ export function finalizeEvaluationReport({ report, results, artifacts, variantNa
     // Gap rate computation runs on every successful report regardless of whether
     // tool trace data is present — text-based signals (markers, hedging) still
     // apply. The samples-file SHA is the mandatory watermark required by spec §7.1.
+    // The same hash watermarks the opt-in holdout breakdown below, so it is computed
+    // once and shared (both coverage-class numbers must carry the same test-set id).
+    const holdoutRatio = report.meta?.request?.holdoutRatio ?? 0;
     const gapReports = computeReportGapRates(report.results, variantNames);
+    const needsWatermark = Object.keys(gapReports).length > 0 || holdoutRatio > 0;
+    const testSetHash = needsWatermark ? computeTestSetHash(samplesPath, samplesSourceFiles) : null;
     if (Object.keys(gapReports).length > 0) {
-        const testSetHash = computeTestSetHash(samplesPath, samplesSourceFiles);
         for (const variant of variantNames) {
             const gr = gapReports[variant];
             if (!gr)
@@ -49,8 +50,18 @@ export function finalizeEvaluationReport({ report, results, artifacts, variantNa
         }
         report.analysis.gapReports = gapReports;
     }
-    if (blind) {
-        applyBlindMode(report, variantNames, `${variantNames.join(',')}:${samplesPath}`);
+    // Opt-in train/holdout generalization breakdown (`omk eval --holdout-ratio`).
+    // Post-hoc over report.results — never perturbs the headline composite or the
+    // bootstrap CI. A large train − holdout gap is the overfitting signal the verdict
+    // overfitting gate reads (src/eval-core/verdict.ts); absent on default runs.
+    if (holdoutRatio > 0) {
+        // 切分按 samples 的稳定原始顺序(文件顺序),不依赖 report.results 的并发完成落盘顺序,
+        // 否则同批样本在不同并发/时序下 holdout 子集会漂、verdict 过拟合门控跟着漂。
+        const sampleIdOrder = samples.map((s) => s.sample_id);
+        const holdout = computeHoldoutBreakdown(report, variantNames, holdoutRatio, sampleIdOrder);
+        holdout.testSetPath = samplesPath;
+        holdout.testSetHash = testSetHash;
+        report.analysis.holdout = holdout;
     }
     return report;
 }

package/dist/eval-workflows/evaluation-pipeline/run-state.d.ts

@@ -22,7 +22,7 @@ export interface EvaluationRunState {
     runningJob: EvaluationJob;
     resolvedJobStore: JobStore | null;
 }
-export declare function initializeEvaluationRunState({ samplesPath, skillDir, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, concurrency, timeoutMs, noCache, blind, project, owner, tags, runId, jobStore, persistJob, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }: {
+export declare function initializeEvaluationRunState({ samplesPath, skillDir, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, concurrency, timeoutMs, noCache, project, owner, tags, runId, jobStore, persistJob, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }: {
     samplesPath: string;
     skillDir: string;
     artifacts: Artifact[];
@@ -34,7 +34,6 @@ export declare function initializeEvaluationRunState({ samplesPath, skillDir, ar
     concurrency: number;
     timeoutMs?: number;
     noCache: boolean;
-    blind: boolean;
     project?: string;
     owner?: string;
     tags?: string[];
@@ -42,6 +41,7 @@ export declare function initializeEvaluationRunState({ samplesPath, skillDir, ar
     jobStore?: JobStore | null;
     persistJob?: boolean;
     repeat?: number;
+    holdoutRatio?: number;
     batch?: boolean;
     judgeRepeat?: number;
     judgeModels?: import('../../types/index.js').JudgeConfig[];

package/dist/eval-workflows/evaluation-pipeline/run-state.js

@@ -14,7 +14,7 @@
 import { buildEvaluationRequest, createFailedJob, createEvaluationRun, createQueuedJob, createSucceededJob, finalizeEvaluationRun, markJobRunning, failEvaluationRun, } from '../../eval-core/evaluation-job.js';
 import { createFileJobStore } from '../../server/job-store.js';
 import { DEFAULT_JOBS_DIR } from '../../eval-core/default-dirs.js';
-export async function initializeEvaluationRunState({ samplesPath, skillDir, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, concurrency, timeoutMs, noCache, blind, project, owner, tags, runId, jobStore, persistJob, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }) {
+export async function initializeEvaluationRunState({ samplesPath, skillDir, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, concurrency, timeoutMs, noCache, project, owner, tags, runId, jobStore, persistJob, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }) {
     const effectiveJudges = judgeModels && judgeModels.length > 0
         ? judgeModels
         : [{ executor: judgeExecutorName, model: judgeModel }];
@@ -29,11 +29,11 @@ export async function initializeEvaluationRunState({ samplesPath, skillDir, arti
         timeoutMs,
         noCache,
         dryRun: false,
-        blind,
         project,
         owner,
         tags,
         repeat,
+        holdoutRatio,
         batch,
         judgeRepeat,
         judgeModels: effectiveJudges,

package/dist/eval-workflows/evaluation-pipeline.d.ts

@@ -47,7 +47,6 @@ export interface EvaluationPipelineOptions {
     project?: string;
     owner?: string;
     tags?: string[];
-    blind?: boolean;
     concurrency?: number;
     timeoutMs?: number;
     noCache?: boolean;
@@ -62,6 +61,8 @@ export interface EvaluationPipelineOptions {
     layeredStats?: boolean;
     /** 透传到 meta.request.repeat */
     repeat?: number;
+    /** 透传到 meta.request.holdoutRatio；> 0 时 report-finalize 算 train/holdout 子集综合分。 */
+    holdoutRatio?: number;
     /** 透传到 meta.request.batch */
     batch?: boolean;
     /** 透传到 meta.request.judgeRepeat 与 grade()，每条 sample × dimension judge N 次 */
@@ -90,7 +91,7 @@ export interface EvaluationPipelineOptions {
     noDiagnostic?: boolean;
 }
 type VariantResult = import('../types/index.js').VariantResult;
-export declare function executeEvaluationPipeline({ samplesPath, samplesBaseDir, samplesSourceFiles, skillDir, samples, tasks, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, executor, judgeExecutor, outputDir, project, owner, tags, blind, concurrency, timeoutMs, noCache, jobStore, persistJob, onProgress, skipConnectivity, verbose, retry, existingResults, requires: _requires, layeredStats, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, runId, lang, effort, noDiagnostic, }: EvaluationPipelineOptions): Promise<{
+export declare function executeEvaluationPipeline({ samplesPath, samplesBaseDir, samplesSourceFiles, skillDir, samples, tasks, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, executor, judgeExecutor, outputDir, project, owner, tags, concurrency, timeoutMs, noCache, jobStore, persistJob, onProgress, skipConnectivity, verbose, retry, existingResults, requires: _requires, layeredStats, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, runId, lang, effort, noDiagnostic, }: EvaluationPipelineOptions): Promise<{
     report: Report;
     filePath: string | null;
 }>;

package/dist/eval-workflows/evaluation-pipeline.js

@@ -30,10 +30,10 @@ import { emitIsolationWarnings, emitPowerWarnings } from './evaluation-pipeline/
 // 兼容 re-export:测试与 run-evaluation.ts 动态 import 仍打 evaluation-pipeline.js
 export { buildPowerWarnings, buildIsolationWarnings } from './evaluation-pipeline/preflight-warnings.js';
 export { _computeTestSetHashForTest } from './evaluation-pipeline/test-set-hash.js';
-export async function executeEvaluationPipeline({ samplesPath, samplesBaseDir, samplesSourceFiles, skillDir, samples, tasks, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, executor, judgeExecutor, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, blind = false, concurrency = 1, timeoutMs, noCache = false, jobStore = null, persistJob = true, onProgress = null, skipConnectivity = false, verbose = false, retry = 0, existingResults, 
+export async function executeEvaluationPipeline({ samplesPath, samplesBaseDir, samplesSourceFiles, skillDir, samples, tasks, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, executor, judgeExecutor, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, concurrency = 1, timeoutMs, noCache = false, jobStore = null, persistJob = true, onProgress = null, skipConnectivity = false, verbose = false, retry = 0, existingResults, 
 // requires 现在由 runEvaluation 上游传给 doctor 处理; eval-pipeline 不再用
 // 但保留接口字段,避免破 programmatic API (类型层面接收, 内部忽略)
-requires: _requires, layeredStats = false, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias = true, budget, strictBaseline, runId, lang = 'zh', effort, noDiagnostic, }) {
+requires: _requires, layeredStats = false, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias = true, budget, strictBaseline, runId, lang = 'zh', effort, noDiagnostic, }) {
     const variantNames = artifacts.map((artifact) => artifact.name);
     const runState = await initializeEvaluationRunState({
         samplesPath,
@@ -47,7 +47,6 @@ requires: _requires, layeredStats = false, repeat, batch, judgeRepeat, judgeMode
         concurrency,
         timeoutMs,
         noCache,
-        blind,
         project,
         owner,
         tags,
@@ -55,6 +54,7 @@ requires: _requires, layeredStats = false, repeat, batch, judgeRepeat, judgeMode
         jobStore,
         persistJob,
         repeat,
+        holdoutRatio,
         batch,
         judgeRepeat,
         judgeModels,
@@ -153,7 +153,6 @@ requires: _requires, layeredStats = false, repeat, batch, judgeRepeat, judgeMode
             results,
             artifacts,
             variantNames,
-            blind,
             samplesPath,
             samplesSourceFiles,
             samples,

package/dist/eval-workflows/run-evaluation.d.ts

@@ -32,6 +32,9 @@ interface CommonEvaluationOptions {
     /** --repeat N. 1 表示单次(默认); > 1 时在 runMultiple 层聚合 variance。
      *  记入 report.meta.request.repeat 让 meta 如实反映用户输入。 */
     repeat?: number;
+    /** --holdout-ratio R. 0 / 缺省 = 不切分(默认)。> 0 时 report-finalize 算 train/holdout
+     *  子集综合分(report.analysis.holdout),供 verdict 过拟合门控读取。 */
+    holdoutRatio?: number;
     /** --batch 模式标记, true 表示当前评测是 skill batch 流程。
      *  记入 report.meta.request.batch。 */
     batch?: boolean;
@@ -46,7 +49,7 @@ interface CommonEvaluationOptions {
     bootstrap?: boolean;
     /** --bootstrap-samples N. Default 1000. */
     bootstrapSamples?: number;
-    /** length-debias toggle. Default true (judge prompt v3-cot-length).
+    /** length-debias toggle. Default true (length-debias instruction on).
      *  CLI passes false when --no-debias-length is set. */
     lengthDebias?: boolean;
     /** hard budget caps. */
@@ -64,7 +67,6 @@ export interface RunEvaluationOptions extends CommonEvaluationOptions {
     skillDir: string;
     variantSpecs?: VariantSpec[];
     dryRun?: boolean;
-    blind?: boolean;
     retry?: number;
     resume?: string;
     /** Explicit persisted run id. Used by batch workflows that need stable child ids. */
@@ -118,7 +120,7 @@ export interface DryRunReport extends DryRunBase {
     samplesPath: string;
     tasks: DryRunTask[];
 }
-export declare function runEvaluation({ samplesPath, skillDir, variantSpecs, model, outputDir, project, owner, tags, noJudge, dryRun, blind, concurrency, timeoutMs, noCache, executorName, jobStore, persistJob, onProgress, skipConnectivity, skipDoctor, lang, mcpConfig, verbose, retry, resume, runId, layeredStats, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, effort, noDiagnostic, }: RunEvaluationOptions): Promise<{
+export declare function runEvaluation({ samplesPath, skillDir, variantSpecs, model, outputDir, project, owner, tags, noJudge, dryRun, concurrency, timeoutMs, noCache, executorName, jobStore, persistJob, onProgress, skipConnectivity, skipDoctor, lang, mcpConfig, verbose, retry, resume, runId, layeredStats, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, effort, noDiagnostic, }: RunEvaluationOptions): Promise<{
     report: Report | DryRunReport;
     filePath: string | null;
 }>;
@@ -134,7 +136,7 @@ export interface DryRunBatchReport extends DryRunBase {
     artifacts: DryRunBatchSkill[];
 }
 export declare function buildVarianceData(runs: Report[], bootstrapSamples?: number, seed?: number): VarianceData | null;
-export declare function runBatchEvaluation({ skillDir, model, outputDir, project, owner, tags, noJudge, dryRun, concurrency, timeoutMs, executorName, jobStore, persistJob, onProgress, onSkillProgress, skipConnectivity, skipDoctor, lang, mcpConfig, verbose, repeat, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, noCache, strictBaseline, variantAllowedSkills, }: RunBatchEvaluationOptions): Promise<{
+export declare function runBatchEvaluation({ skillDir, model, outputDir, project, owner, tags, noJudge, dryRun, concurrency, timeoutMs, executorName, jobStore, persistJob, onProgress, onSkillProgress, skipConnectivity, skipDoctor, lang, mcpConfig, verbose, repeat, holdoutRatio, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, noCache, strictBaseline, variantAllowedSkills, }: RunBatchEvaluationOptions): Promise<{
     report: BatchEvaluationReport | DryRunBatchReport;
     filePath: string | null;
 }>;

package/dist/eval-workflows/run-evaluation.js

@@ -8,7 +8,7 @@ import { buildDryRunTaskReport, prepareEvaluationRun, } from './evaluation-prepa
 import { executeEvaluationPipeline } from './evaluation-pipeline.js';
 import { findSaturationPoint } from '../analysis/saturation.js';
 import { bootstrapMeanCI, DEFAULT_BOOTSTRAP_ALPHA, DEFAULT_BOOTSTRAP_SAMPLES } from '../eval-core/bootstrap.js';
-export async function runEvaluation({ samplesPath, skillDir, variantSpecs = [], model = DEFAULT_MODEL, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, noJudge = false, dryRun = false, blind = false, concurrency = 1, timeoutMs, noCache = false, executorName = 'claude', jobStore = null, persistJob = true, onProgress = null, skipConnectivity = false, skipDoctor = false, lang = 'zh', mcpConfig, verbose = false, retry = 0, resume, runId, layeredStats = false, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, effort, noDiagnostic, }) {
+export async function runEvaluation({ samplesPath, skillDir, variantSpecs = [], model = DEFAULT_MODEL, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, noJudge = false, dryRun = false, concurrency = 1, timeoutMs, noCache = false, executorName = 'claude', jobStore = null, persistJob = true, onProgress = null, skipConnectivity = false, skipDoctor = false, lang = 'zh', mcpConfig, verbose = false, retry = 0, resume, runId, layeredStats = false, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, effort, noDiagnostic, }) {
     // Unified judgeModels → derive single-judge fields for downstream pipeline / grading
     // (which still operate on string `judgeModel` + `judgeExecutorName` fields per call).
     const effectiveJudgeModels = judgeModels && judgeModels.length > 0
@@ -152,7 +152,6 @@ export async function runEvaluation({ samplesPath, skillDir, variantSpecs = [],
         project,
         owner,
         tags,
-        blind,
         concurrency,
         timeoutMs,
         noCache,
@@ -166,6 +165,7 @@ export async function runEvaluation({ samplesPath, skillDir, variantSpecs = [],
         requires,
         layeredStats,
         repeat,
+        holdoutRatio,
         batch,
         judgeRepeat,
         judgeModels,
@@ -367,7 +367,7 @@ export function buildVarianceData(runs, bootstrapSamples = DEFAULT_BOOTSTRAP_SAM
         ...(saturation ? { saturation } : {}),
     };
 }
-export async function runBatchEvaluation({ skillDir, model = DEFAULT_MODEL, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, noJudge = false, dryRun = false, concurrency = 1, timeoutMs, executorName = 'claude', jobStore = null, persistJob = true, onProgress = null, onSkillProgress = null, skipConnectivity = false, skipDoctor = false, lang = 'zh', mcpConfig, verbose = false, repeat, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, noCache = false, strictBaseline, variantAllowedSkills, }) {
+export async function runBatchEvaluation({ skillDir, model = DEFAULT_MODEL, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, noJudge = false, dryRun = false, concurrency = 1, timeoutMs, executorName = 'claude', jobStore = null, persistJob = true, onProgress = null, onSkillProgress = null, skipConnectivity = false, skipDoctor = false, lang = 'zh', mcpConfig, verbose = false, repeat, holdoutRatio, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, noCache = false, strictBaseline, variantAllowedSkills, }) {
     // Same unified judge derivation as runEvaluation (downstream pipeline / report build
     // still uses single judgeModel + judgeExecutorName per call).
     const effectiveJudgeModels = judgeModels && judgeModels.length > 0
@@ -486,12 +486,14 @@ export async function runBatchEvaluation({ skillDir, model = DEFAULT_MODEL, outp
         strictBaseline,
         variantAllowedSkills,
         runSingleEvaluation: async (options) => {
-            // repeat > 1 时走 runMultiple 做 variance; batch=true 标记让 meta.request 如实反映
+            // repeat > 1 时走 runMultiple 做 variance; batch=true 标记让 meta.request 如实反映。
+            // holdoutRatio 显式转发(executeBatchEvaluationRuns 不一定把它塞进 options),否则
+            // batch 子报告 meta.request.holdoutRatio 丢失、过拟合门控对 batch 静默失效。
             if (repeat && repeat > 1) {
-                const multi = await runMultiple({ ...options, repeat, batch: true, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget });
+                const multi = await runMultiple({ ...options, repeat, holdoutRatio, batch: true, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget });
                 return { report: multi.report, filePath: multi.filePath };
             }
-            const result = await runEvaluation({ ...options, batch: true, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget });
+            const result = await runEvaluation({ ...options, batch: true, holdoutRatio, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget });
             return { report: result.report, filePath: result.filePath };
         },
     });

package/dist/executors/claude-cli.js

@@ -2,20 +2,19 @@ import { extractAgentTrace, isClaudeSdkResultMessage } from './claude-sdk-trace.
 import { buildExecEnv, DEFAULT_TIMEOUT_MS, errorMessage, interruptedExecResult, MAX_BUFFER, spawnWithSigintPropagation, timeoutExecResult, } from './shared.js';
 import { materializeForCliConfigDir } from '../eval-core/mocks-runtime.js';
 // claude CLI 用 `--disable-slash-commands` (文档:"Disable all skills") +
-// `--disallowedTools Skill` 实现与 SDK 等价的完全隔离。但 CLI 没有 partial whitelist
-// 的 flag,所以 [name1, ...] 必须 throw。
+// `--disallowedTools Skill` 实现与 SDK 等价的完全隔离。非空 skill 白名单不再支持
+// (它无法真正隔离,见 claude-sdk.ts buildSdkIsolationOptions),所以 [name1, ...] 必须 throw。
 //
 //   undefined           → 不传任何 flag(原行为,全发现)
 //   []                  → --disable-slash-commands + --disallowedTools Skill
 //                         (main session skill discovery + subagent Skill 工具调用都堵)
-//   [...] (length > 0)  → throw,提示用 claude-sdk executor 走精准白名单
+//   [...] (length > 0)  → throw,非空白名单已移除
 function applySkillIsolationToCliArgs(args, allowedSkills) {
     if (allowedSkills === undefined)
         return;
     if (allowedSkills.length > 0) {
-        throw new Error(`claude-cli executor 不支持 partial skill 白名单(allowedSkills=${JSON.stringify(allowedSkills)})。\n`
-            + `  仅支持 [](映射为 --disable-slash-commands + --disallowedTools Skill)或 undefined(默认)。\n`
-            + `  精确白名单请改用 --executor claude-sdk(SDK skills option pass-through)。`);
+        throw new Error(`skill 白名单(allowedSkills=${JSON.stringify(allowedSkills)})不再支持:非空白名单无法真正隔离。\n`
+            + `  仅支持 [](映射为 --disable-slash-commands + --disallowedTools Skill,全封死)或 undefined(不隔离)。`);
     }
     // 完全隔离:双堵 main session skill 发现 + subagent Skill 工具
     args.push('--disable-slash-commands', '--disallowedTools', 'Skill');

package/dist/executors/claude-sdk.d.ts

@@ -3,8 +3,11 @@ import type { ExecResult, ExecutorInput } from '../types/index.js';
  * Map ExecutorInput.allowedSkills to SDK query options for skill isolation.
  *   undefined → {} (SDK default: full ~/.claude/skills/ discovery)
  *   []        → { skills: [], disallowedTools: ['Skill'] } (main session + subagent 双堵)
- *   [...]     → { skills: [...] } (main session whitelist; subagent 走独立 channel,
- *               白名单场景 v1 不强制 subagent 跟随)
+ *   [...]     → throw(非空 skill 白名单不再支持:它从不能真隔离 —— 主会话 skill 发现虽被
+ *               `skills:[...]` 收窄,但子代理 Skill 工具与 cwd 文件系统两条 channel 封不住,
+ *               会产出看着干净、实则被白名单外 skill 污染的报告。隔离只留两档:undefined
+ *               (不隔离)与 [](全封死),与 claude-cli / codex-cli 一致;多 skill 组合实验
+ *               请控制评测环境而非靠白名单。)
  *
  * Exported for unit tests to lock the option-shape contract.
  */

package/dist/executors/claude-sdk.js

@@ -9,8 +9,11 @@ let sdkQuery = null;
  * Map ExecutorInput.allowedSkills to SDK query options for skill isolation.
  *   undefined → {} (SDK default: full ~/.claude/skills/ discovery)
  *   []        → { skills: [], disallowedTools: ['Skill'] } (main session + subagent 双堵)
- *   [...]     → { skills: [...] } (main session whitelist; subagent 走独立 channel,
- *               白名单场景 v1 不强制 subagent 跟随)
+ *   [...]     → throw(非空 skill 白名单不再支持:它从不能真隔离 —— 主会话 skill 发现虽被
+ *               `skills:[...]` 收窄,但子代理 Skill 工具与 cwd 文件系统两条 channel 封不住,
+ *               会产出看着干净、实则被白名单外 skill 污染的报告。隔离只留两档:undefined
+ *               (不隔离)与 [](全封死),与 claude-cli / codex-cli 一致;多 skill 组合实验
+ *               请控制评测环境而非靠白名单。)
  *
  * Exported for unit tests to lock the option-shape contract.
  */
@@ -19,7 +22,8 @@ export function buildSdkIsolationOptions(allowedSkills) {
         return {};
     if (allowedSkills.length === 0)
         return { skills: allowedSkills, disallowedTools: ['Skill'] };
-    return { skills: allowedSkills };
+    throw new Error(`skill 白名单(allowedSkills=${JSON.stringify(allowedSkills)})不再支持:非空白名单无法真正隔离`
+        + `(子代理 Skill 工具 + cwd 文件系统两条 channel 封不住)。仅支持 [](全封死)或 undefined(不隔离)。`);
 }
 async function getSdkQuery() {
     if (!sdkQuery) {
@@ -29,6 +33,12 @@ async function getSdkQuery() {
     return sdkQuery;
 }
 export async function claudeSdkExecutor({ model, system, prompt, cwd, skillDir, timeoutMs = DEFAULT_TIMEOUT_MS, verbose = false, allowedSkills, mocks, mocksBaseDir, mocksStrict, lean, effort }) {
+    // 隔离选项在 timer / try 之前解析:非空 allowedSkills(不再支持的 skill 白名单)必须在这里
+    // fail-fast 抛错,而不是被下面的 catch 吞成 ok:false 的 ExecResult(那会把配置错误伪装成
+    // 每个 sample 执行失败、产出全失败报告,与 claude-cli / codex 的硬抛口径不一致)。lean 仍
+    // 覆盖为硬堵,但 buildSdkIsolationOptions 先跑一遍,故 lean 也无法绕过非空校验。
+    const baseIsolationOpts = buildSdkIsolationOptions(allowedSkills);
+    const isolationOpts = lean ? { skills: [], disallowedTools: ['*'] } : baseIsolationOpts;
     const start = Date.now();
     const abortController = new AbortController();
     const timer = setTimeout(() => abortController.abort(), timeoutMs);
@@ -45,11 +55,6 @@ export async function claudeSdkExecutor({ model, system, prompt, cwd, skillDir,
     const mockStatsOf = () => hookHandle ? { ...hookHandle.stats } : undefined;
     try {
         const query = await getSdkQuery();
-        // lean 模式:纯文本生成路径,不需要工具循环 / skill 发现,
-        // 用 disallowedTools:['*'] + skills:[] 直接堵住,优先级高于 isolationOpts。
-        const isolationOpts = lean
-            ? { skills: [], disallowedTools: ['*'] }
-            : buildSdkIsolationOptions(allowedSkills);
         // effort:lean 强制 'low'(生成路径不需要思考),否则透传调用方传入。
         // SDK 暴露 EffortLevel = 'low' | 'medium' | 'high' | 'xhigh' | 'max',直接对应。
         const effectiveEffort = lean ? 'low' : effort;

package/dist/executors/codex-cli.js

@@ -11,14 +11,13 @@ import { buildExecEnv, DEFAULT_TIMEOUT_MS, errorMessage, interruptedExecResult,
 //   undefined         → 不传 -C(原行为,看 cwd 里有什么 codex 自己决定)
 //   []                → 必须提供 cwd 非空(否则 throw),caller 应传一个
 //                       isolated 空目录(如 ~/.oh-my-knowledge/state/isolated-cwd/)
-//   [...] (length>0)  → throw,codex CLI 没有 partial 白名单 flag
+//   [...] (length>0)  → throw,非空白名单已移除(无法真正隔离)
 export function isolateCodexCwd(allowedSkills, cwd, executorName = 'codex-cli') {
     if (allowedSkills === undefined)
         return;
     if (allowedSkills.length > 0) {
-        throw new Error(`${executorName} executor 不支持 partial skill 白名单(allowedSkills=${JSON.stringify(allowedSkills)})。\n`
-            + `  仅支持 [](强制 cwd 隔离,需提供 cwd 非空)或 undefined(默认)。\n`
-            + `  codex CLI 无 partial 白名单 flag,请改用其他 executor 或显式 cwd 隔离。`);
+        throw new Error(`skill 白名单(allowedSkills=${JSON.stringify(allowedSkills)})不再支持:非空白名单无法真正隔离。\n`
+            + `  仅支持 [](强制 cwd 隔离,需提供 cwd 非空,全封死)或 undefined(不隔离)。`);
     }
     // allowedSkills === [] 时必须有 cwd(channel 3 cwd 隔离是 codex 唯一 channel)
     if (!cwd) {

package/dist/executors/shared.d.ts

@@ -5,6 +5,8 @@ export declare const DEFAULT_MODEL = "sonnet";
 export declare const JUDGE_MODEL = "haiku";
 export declare const DEFAULT_TIMEOUT_MS = 600000;
 export declare const MAX_BUFFER: number;
+export type ExecutorVendor = 'anthropic' | 'openai' | 'google' | 'unknown';
+export declare function executorVendor(executor: string): ExecutorVendor;
 export interface TokenUsage {
     input_tokens?: number;
     output_tokens?: number;

package/dist/executors/shared.js

@@ -9,6 +9,21 @@ export const DEFAULT_MODEL = 'sonnet';
 export const JUDGE_MODEL = 'haiku';
 export const DEFAULT_TIMEOUT_MS = 600_000;
 export const MAX_BUFFER = 10 * 1024 * 1024;
+// 把 executor 名归到模型厂商家族,用于评委独立性判断(同厂商评委 = 自我偏好敞口)。
+// codex 是 OpenAI 的代码模型,故 codex* 归 OpenAI。自定义 script executor 无法归类 → unknown
+// (调用方据此跳过判断,不误报)。
+const EXECUTOR_VENDOR = {
+    claude: 'anthropic',
+    'claude-sdk': 'anthropic',
+    'anthropic-api': 'anthropic',
+    codex: 'openai',
+    'codex-sdk': 'openai',
+    'openai-api': 'openai',
+    gemini: 'google',
+};
+export function executorVendor(executor) {
+    return EXECUTOR_VENDOR[executor] ?? 'unknown';
+}
 export function asErrorLike(err) {
     return typeof err === 'object' && err !== null ? err : {};
 }

package/dist/grading/assertions.js

@@ -1,6 +1,7 @@
 import { resolve } from 'node:path';
 import _Ajv from 'ajv';
 import { ASSERTION_LAYER } from './layered-scores.js';
+import { buildSemanticSimilarityPrompt, SEMANTIC_SIMILARITY_SYSTEM, buildRagJudgePrompt } from '../shared/llm-prompts/judge-prompts.js';
 const Ajv = _Ajv.default ?? _Ajv;
 const ajv = new Ajv();
 const CUSTOM_ASSERTION_TIMEOUT_MS = 30_000;
@@ -332,24 +333,10 @@ export async function runAsyncAssertions(output, assertions, { executor, judgeMo
         let message = '';
         if (assertion.type === 'semantic_similarity') {
             const reference = assertion.reference || '';
-            const judgePrompt = [
-                '请判断以下两段文本的语义相似度。',
-                '',
-                '## 参考文本',
-                reference,
-                '',
-                '## 待评估文本',
-                output,
-                '',
-                '请返回 JSON（不要包含 markdown 代码块标记）：',
-                '{"score": <1-5的整数>, "reason": "<简短理由>"}',
-                '',
-                '评分：1=完全无关, 2=略有关联, 3=部分相似, 4=大致相同, 5=高度一致',
-            ].join('\n');
             const result = await executor({
                 model: judgeModel,
-                system: '你是语义相似度评审员。只返回 JSON，不要其他内容。',
-                prompt: judgePrompt,
+                system: SEMANTIC_SIMILARITY_SYSTEM,
+                prompt: buildSemanticSimilarityPrompt(reference, output),
             });
             asyncCostUSD += result.costUSD || 0;
             if (result.costReportedByExecutor === false)
@@ -423,40 +410,6 @@ export async function runAsyncAssertions(output, assertions, { executor, judgeMo
         ...(anyCostUnreported && { judgeCostReportedByExecutor: false }),
     };
 }
-// ===========================================================================
-// RAG-specific judge metrics
-// ===========================================================================
-//
-// Three metrics, all running through the LLM judge:
-//
-//  - faithfulness:      does the output's content stay grounded in the
-//                       reference context? Anti-hallucination check.
-//  - answer_relevancy:  does the output directly answer the user's question?
-//                       Catches verbose dodges and topic drift.
-//  - context_recall:    are the key facts from the gold context actually
-//                       used in the output? Catches retrieved-but-ignored
-//                       context (a common RAG bug).
-//
-// Implementation notes:
-//
-//  1. Each prompt is a SINGLE-CALL judge (1-5 score) rather than the multi-step
-//     statement-decomposition that RAGAS uses. This is honest tradeoff: simpler,
-//     faster, less rigorous than RAGAS but consistent with omk's other LLM-judge
-//     assertions. Users who need RAGAS-grade decomposition can drop down to a
-//     custom assertion.
-//  2. The prompt includes the SAME length-debias paragraph as the main judge
-//     prompt (v3-cot-length) — output verbosity is not a quality signal here
-//     either. This is the "auto-inherit length-debias" claim from the  plan.
-//  3. Reference resolution:
-//       faithfulness:    sample.context  (or assertion.reference override)
-//       context_recall:  assertion.reference (or sample.context fallback)
-//       answer_relevancy: sample.prompt — no reference needed
-//  4. Threshold defaults to 3 (same as semantic_similarity). User can override.
-const RAG_LENGTH_DEBIAS = [
-    '## 重要:长度不是质量信号',
-    '评分时聚焦内容实质,不要因输出更长就给更高分。',
-    '简洁正确的回答与冗长正确的回答应得相同分数。',
-].join('\n');
 async function runRagJudge(assertion, output, sample, executor, judgeModel) {
     const threshold = assertion.threshold ?? 3;
     let prompt;
@@ -466,55 +419,10 @@ async function runRagJudge(assertion, output, sample, executor, judgeModel) {
         if (!context) {
             return { passed: false, message: 'faithfulness: 缺少 sample.context 或 assertion.reference', costUSD: 0 };
         }
-        system = '你是 RAG 评审员,专注判断输出是否被参考 context 支持。只返回 JSON。';
-        prompt = [
-            '请判断"待评估输出"中的事实性陈述是否被"参考 context"支持。',
-            '',
-            '## 参考 context',
-            context,
-            '',
-            '## 待评估输出',
-            output,
-            '',
-            RAG_LENGTH_DEBIAS,
-            '',
-            '## 评分流程',
-            '1. 列出待评估输出中所有事实性陈述',
-            '2. 逐条判断是否能在 context 中找到支持',
-            '3. 给出 1-5 分:',
-            '   5 = 全部陈述都有 context 支持,无编造',
-            '   4 = 多数有支持,有 1-2 处不重要的编造',
-            '   3 = 一半有支持',
-            '   2 = 多数无支持',
-            '   1 = 完全编造或与 context 矛盾',
-            '',
-            '请返回 JSON(不要 markdown 代码块):',
-            '{"score": <1-5的整数>, "reason": "<简短理由>"}',
-        ].join('\n');
+        ({ system, prompt } = buildRagJudgePrompt('faithfulness', { output, context }));
     }
     else if (assertion.type === 'answer_relevancy') {
-        system = '你是答题切题度评审员。只返回 JSON。';
-        prompt = [
-            '请判断"AI 输出"是否直接、切题地回答了"用户问题"。',
-            '',
-            '## 用户问题',
-            sample.prompt,
-            '',
-            '## AI 输出',
-            output,
-            '',
-            RAG_LENGTH_DEBIAS,
-            '',
-            '## 评分',
-            '5 = 完整切题回答,无冗余无遗漏',
-            '4 = 切题但有少量冗余或小遗漏',
-            '3 = 部分切题,部分跑题或避而不答',
-            '2 = 大部分跑题',
-            '1 = 完全跑题或拒答',
-            '',
-            '请返回 JSON(不要 markdown 代码块):',
-            '{"score": <1-5的整数>, "reason": "<简短理由>"}',
-        ].join('\n');
+        ({ system, prompt } = buildRagJudgePrompt('answer_relevancy', { output, question: sample.prompt }));
     }
     else {
         // context_recall
@@ -522,31 +430,7 @@ async function runRagJudge(assertion, output, sample, executor, judgeModel) {
         if (!reference) {
             return { passed: false, message: 'context_recall: 缺少 assertion.reference 或 sample.context', costUSD: 0 };
         }
-        system = '你是 context 覆盖率评审员。只返回 JSON。';
-        prompt = [
-            '请判断"参考 gold"中的关键事实在"AI 输出"中被覆盖的程度。',
-            '',
-            '## 参考 gold',
-            reference,
-            '',
-            '## AI 输出',
-            output,
-            '',
-            RAG_LENGTH_DEBIAS,
-            '',
-            '## 评分流程',
-            '1. 列出参考中的关键事实(忽略修饰性内容)',
-            '2. 检查每条是否在输出中被提及/使用',
-            '3. 给出 1-5 分:',
-            '   5 = 全部关键事实被覆盖',
-            '   4 = 大部分覆盖,缺 1-2 条次要事实',
-            '   3 = 一半覆盖',
-            '   2 = 仅覆盖少量',
-            '   1 = 完全未覆盖',
-            '',
-            '请返回 JSON(不要 markdown 代码块):',
-            '{"score": <1-5的整数>, "reason": "<简短理由>"}',
-        ].join('\n');
+        ({ system, prompt } = buildRagJudgePrompt('context_recall', { output, reference }));
     }
     const result = await executor({ model: judgeModel, system, prompt });
     const reported = result.costReportedByExecutor === false ? { costReportedByExecutor: false } : {};

package/dist/grading/gold-cli.js

@@ -93,7 +93,7 @@ export function formatGoldCompare(result, gold) {
             lines.push(`  报告缺失:         ${result.missing.join(', ')}`);
         return lines.join('\n');
     }
-    lines.push(`  Krippendorff α:   ${fmt(a.alpha)}   (主指标，序数加权)`);
+    lines.push(`  Krippendorff α:   ${fmt(a.alpha)}   (主指标，区间加权)`);
     lines.push(`  α 95% CI:         [${fmt(a.alphaCI.low)}, ${fmt(a.alphaCI.high)}]`);
     lines.push(`  加权 κ:           ${fmt(a.weightedKappa)}   (副指标)`);
     lines.push(`  Pearson r:        ${fmt(a.pearson)}   (仅查 rank order)`);

package/dist/grading/human-gold.d.ts

@@ -15,9 +15,11 @@
  *
  * Three metrics are exported:
  *
- *  - **Krippendorff's α (interval)** — primary. Distribution-free, supports
- *    ordinal/interval scales naturally, doesn't assume coders are exchangeable
- *    (good fit when one "coder" is a model and the other a human annotator).
+ *  - **Krippendorff's α (interval weights)** — primary. Distribution-free,
+ *    doesn't assume coders are exchangeable (good fit when one "coder" is a model
+ *    and the other a human annotator). Uses interval distance δ²=(c−k)² — a
+ *    defensible choice for 1-5 Likert; an ordinal-distance variant would change α
+ *    (BREAKING-COMPARABILITY) and is not implemented here.
  *  - **Quadratic-weighted Cohen's κ** — secondary. Familiar to many readers,
  *    useful as a sanity check. Reports lower than α when marginals diverge.
  *  - **Pearson r** — tertiary. Captures rank-order agreement only; doesn't

package/dist/grading/human-gold.js

@@ -15,9 +15,11 @@
  *
  * Three metrics are exported:
  *
- *  - **Krippendorff's α (interval)** — primary. Distribution-free, supports
- *    ordinal/interval scales naturally, doesn't assume coders are exchangeable
- *    (good fit when one "coder" is a model and the other a human annotator).
+ *  - **Krippendorff's α (interval weights)** — primary. Distribution-free,
+ *    doesn't assume coders are exchangeable (good fit when one "coder" is a model
+ *    and the other a human annotator). Uses interval distance δ²=(c−k)² — a
+ *    defensible choice for 1-5 Likert; an ordinal-distance variant would change α
+ *    (BREAKING-COMPARABILITY) and is not implemented here.
  *  - **Quadratic-weighted Cohen's κ** — secondary. Familiar to many readers,
  *    useful as a sanity check. Reports lower than α when marginals diverge.
  *  - **Pearson r** — tertiary. Captures rank-order agreement only; doesn't