oh-my-knowledge 0.32.0 → 0.34.0

package/dist/eval-core/verdict.js

@@ -6,9 +6,10 @@
  * v0.21 makes the data trustworthy.  closes the last mile: senior
  * engineers' #1 complaint is "the report has all the right numbers but I
  * still need 30 minutes to read it before I can decide". This module
- * aggregates the four data sources omk already produces — Bootstrap CI on
- * the pairwise diff, three-layer CI gate, saturation curve, and (when
- * available) Krippendorff α against gold — and emits one of four verdicts:
+ * aggregates the data sources omk already produces — Bootstrap CI on the
+ * pairwise diff, three-layer CI gate, saturation curve, inter-judge ensemble
+ * agreement, and (when available) Krippendorff α against gold — and emits one
+ * of six verdicts:
  *
  *  - **PROGRESS**     diff CI shows real positive shift, no layer regressed
  *  - **CAUTIOUS**     positive shift but at least one layer broke its gate,
@@ -27,6 +28,23 @@
  * empirical) is documented inline so users can audit and override.
  */
 import { evaluateLayerGates } from './layer-gates.js';
+/**
+ * Below this sample count a non-significant diff is read as UNDERPOWERED
+ * (only large effects are detectable) rather than NOISE. Matches the
+ * pre-flight power band documented in `docs/specs/sample-design-spec.md`;
+ * guarded by `test/scripts/doc-constants-drift.test.ts`.
+ */
+export const UNDERPOWERED_MIN_SAMPLES = 20;
+/**
+ * Inter-judge Pearson bands, shared with the report's ensemble-agreement audit
+ * badge (`src/renderer/summary.ts`) so the verdict gate and the UI read one scale.
+ *  - `>= ENSEMBLE_STRONG_PEARSON` (0.7): judges agree on rank order (badge green).
+ *  - `< ENSEMBLE_DISSENT_PEARSON` (0.4): strong disagreement (badge red). A
+ *    would-be PROGRESS is downgraded to CAUTIOUS — the judge-layer signal driving
+ *    the win is unreliable when the ensemble can't agree.
+ */
+export const ENSEMBLE_STRONG_PEARSON = 0.7;
+export const ENSEMBLE_DISSENT_PEARSON = 0.4;
 /**
  * Compute a verdict for a finished report. Pure function — no I/O.
  */
@@ -133,7 +151,7 @@ function verdictForPair(pair, summary, sampleCount, report, gateThreshold, trivi
         // Diff CI contains 0. Distinguish "underpowered (saturation says: more samples needed)"
         // from "noise (saturation says: we're saturated, the effect just isn't there)".
         const satVerdict = report.variance?.saturation?.verdicts?.[treatment];
-        const underpowered = sampleCount < 20 ||
+        const underpowered = sampleCount < UNDERPOWERED_MIN_SAMPLES ||
             (satVerdict && !satVerdict.saturated && satVerdict.confidence !== 'high');
         if (underpowered) {
             return {
@@ -167,6 +185,19 @@ function verdictForPair(pair, summary, sampleCount, report, gateThreshold, trivi
             headline: `${headlineCore} · significant but practically tiny`,
         };
     }
+    // Ensemble dissent gate: a real, gate-passing gain is only PROGRESS if the
+    // judges that produced it actually agree. When inter-judge Pearson sits in the
+    // badge's red band (< ENSEMBLE_DISSENT_PEARSON) for either compared variant,
+    // the judge-layer signal behind the win is unreliable → downgrade to CAUTIOUS.
+    const dissent = ensembleDissent(pair, summary);
+    if (dissent) {
+        return {
+            control,
+            treatment,
+            level: 'CAUTIOUS',
+            headline: `${headlineCore} · gain real, but judges disagree on ${dissent.variant} (Pearson ${dissent.pearson} < ${ENSEMBLE_DISSENT_PEARSON})`,
+        };
+    }
     // Did control break a gate? Then treatment winning isn't surprising — flag.
     if (!cGate.allPass) {
         return {
@@ -178,6 +209,25 @@ function verdictForPair(pair, summary, sampleCount, report, gateThreshold, trivi
     }
     return { control, treatment, level: 'PROGRESS', headline: `${headlineCore} · clean win` };
 }
+/**
+ * Strong inter-judge disagreement on the compared pair, if any. Reuses the
+ * persisted per-variant ensemble Pearson (`summary[v].judgeAgreement.pearson`).
+ * Returns the worst offender below the dissent threshold, or null when every
+ * variant agrees / has no ensemble signal (single- or constant-score judge ⇒
+ * pearson undefined, nothing to act on — its disagreement, if any, still shows
+ * in the report's MAD column).
+ */
+function ensembleDissent(pair, summary) {
+    let worst = null;
+    for (const variant of [pair.treatment, pair.control]) {
+        const pearson = summary[variant]?.judgeAgreement?.pearson;
+        if (pearson != null && pearson < ENSEMBLE_DISSENT_PEARSON) {
+            if (!worst || pearson < worst.pearson)
+                worst = { variant, pearson };
+        }
+    }
+    return worst;
+}
 function avgComposite(s) {
     if (!s)
         return 0;
@@ -319,4 +369,3 @@ export function formatVerdictText(result, options = {}) {
     }
     return lines.join('\n');
 }
-//# sourceMappingURL=verdict.js.map

package/dist/eval-workflows/batch-evaluation-workflow.d.ts

@@ -1,8 +1,12 @@
-import type { Artifact, BatchEvaluationReport, EvaluationReport, JobStore, ProgressCallback } from '../types/index.js';
+import type { BatchEvaluationReport, EvaluationReport, JobStore, ProgressCallback, VariantSpec } from '../types/index.js';
 interface RunSingleEvaluationOptions {
     samplesPath: string;
     skillDir: string;
-    artifacts: Artifact[];
+    /** 每个 batch entry 的实验结构(baseline control vs 当前 skill treatment),由
+     *  runEvaluation 的 spec-based 解析统一绑定 role / 隔离。 */
+    variantSpecs: VariantSpec[];
+    /** strict-baseline default,透传给 per-skill runEvaluation(baseline → allowedSkills=[])。 */
+    strictBaseline?: boolean;
     model: string;
     judgeModel: string;
     outputDir: string | null;
@@ -42,6 +46,18 @@ interface CompletedBatchSkillRun {
     report: EvaluationReport;
     filePath: string | null;
 }
+/** Batch 每个 entry 的实验结构固定:baseline control vs 当前 skill treatment。
+ *  构造 VariantSpec 交给 runEvaluation 的 spec-based 解析按身份绑定 role / 隔离——与单跑
+ *  路径同一处绑定逻辑,batch 不再自管 artifact 拼装(此前两处手抄 resolveArtifacts().map()
+ *  导致 #183 角色误绑各存一份)。
+ *  两个 variant 的 allowedSkills 都从 eval.yaml variants[].allowedSkills 取:treatment 按 skill
+ *  名 entry.name 查、baseline 按保留名 `baseline` 查,挂到对应 spec 上由 prepareEvaluationRun
+ *  统一绑定。baseline 的显式声明(白名单或 `[]`)必须保留——eval.yaml variant.allowedSkills
+ *  优先于 strictBaseline 默认,漏挂会让 `--batch --config` 的 baseline 隔离配置静默失效。 */
+export declare function buildBatchVariantSpecs(entry: {
+    name: string;
+    skillPath: string;
+}, variantAllowedSkills?: Record<string, string[]>): VariantSpec[];
 export declare function buildBatchEvaluationReport({ batchRunId, skillDir, skillEntries, skillResults, model, judgeModel, noJudge, executorName, judgeExecutorName, project, owner, tags, concurrency, timeoutMs, totalCostUSD, repeat, judgeModels, noCache, bootstrap, bootstrapSamples, lengthDebias, budget, }: {
     batchRunId: string;
     skillDir: string;
@@ -127,4 +143,3 @@ export declare function executeBatchEvaluationRuns({ skillDir, skillEntries, mod
     filePath: string | null;
 }>;
 export {};
-//# sourceMappingURL=batch-evaluation-workflow.d.ts.map

package/dist/eval-workflows/batch-evaluation-workflow.js

@@ -1,9 +1,34 @@
-import { dirname, resolve } from 'node:path';
+import { dirname } from 'node:path';
 import { DEFAULT_OUTPUT_DIR, generateRunId, getCliVersion, getGitInfo, persistReport } from '../eval-core/evaluation-reporting.js';
 import { buildEvaluationRequest, createEvaluationRun, createSucceededJob, finalizeEvaluationRun } from '../eval-core/evaluation-job.js';
 import { getExecutorRuntimeFingerprint } from '../executors/runtime-fingerprint.js';
 import { createFileJobStore, DEFAULT_JOBS_DIR } from '../server/job-store.js';
-import { resolveArtifacts } from '../inputs/skill-loader.js';
+/** Batch 每个 entry 的实验结构固定:baseline control vs 当前 skill treatment。
+ *  构造 VariantSpec 交给 runEvaluation 的 spec-based 解析按身份绑定 role / 隔离——与单跑
+ *  路径同一处绑定逻辑,batch 不再自管 artifact 拼装(此前两处手抄 resolveArtifacts().map()
+ *  导致 #183 角色误绑各存一份)。
+ *  两个 variant 的 allowedSkills 都从 eval.yaml variants[].allowedSkills 取:treatment 按 skill
+ *  名 entry.name 查、baseline 按保留名 `baseline` 查,挂到对应 spec 上由 prepareEvaluationRun
+ *  统一绑定。baseline 的显式声明(白名单或 `[]`)必须保留——eval.yaml variant.allowedSkills
+ *  优先于 strictBaseline 默认,漏挂会让 `--batch --config` 的 baseline 隔离配置静默失效。 */
+export function buildBatchVariantSpecs(entry, variantAllowedSkills) {
+    const baselineAllowed = variantAllowedSkills?.baseline;
+    const treatmentAllowed = variantAllowedSkills?.[entry.name];
+    return [
+        {
+            name: 'baseline',
+            role: 'control',
+            expr: 'baseline',
+            ...(baselineAllowed !== undefined && { allowedSkills: baselineAllowed }),
+        },
+        {
+            name: entry.name,
+            role: 'treatment',
+            expr: entry.skillPath,
+            ...(treatmentAllowed !== undefined && { allowedSkills: treatmentAllowed }),
+        },
+    ];
+}
 function commonRuntime(runtimes) {
     const values = Object.values(runtimes);
     if (values.length === 0)
@@ -110,7 +135,7 @@ export function buildBatchEvaluationReport({ batchRunId, skillDir, skillEntries,
         ...(noJudge ? {} : { runtime: getExecutorRuntimeFingerprint(jc.executor, jc.model, { skillDir }) }),
     }));
     const report = {
-        kind: 'batch-evaluation',
+        reportKind: 'batch-evaluation',
         id: batchRunId,
         mode: 'skill',
         meta: {
@@ -146,21 +171,13 @@ export async function executeBatchEvaluationRuns({ skillDir, skillEntries, model
     for (let i = 0; i < skillEntries.length; i++) {
         const entry = skillEntries[i];
         onSkillProgress?.({ phase: 'start', skill: entry.name, current: i + 1, total: skillEntries.length });
-        // Batch mode 的实验结构固定为 baseline control vs 当前 skill treatment。
-        const perSkillAllowedSkills = variantAllowedSkills?.[entry.name] !== undefined
-            ? { ...variantAllowedSkills, [entry.skillPath]: variantAllowedSkills[entry.name] }
-            : variantAllowedSkills;
-        const skillArtifacts = resolveArtifacts(resolve(skillDir), ['baseline', entry.skillPath], { strictBaseline, variantAllowedSkills: perSkillAllowedSkills }).map((artifact) => {
-            if (artifact.name === entry.skillPath) {
-                return { ...artifact, name: entry.name, experimentRole: 'treatment' };
-            }
-            return { ...artifact, experimentRole: 'control' };
-        });
+        const variantSpecs = buildBatchVariantSpecs(entry, variantAllowedSkills);
         const childRunId = `${batchRunId}-${String(i + 1).padStart(2, '0')}-${safeRunIdPart(entry.name)}`;
         const { report, filePath } = await runSingleEvaluation({
             samplesPath: entry.samplesPath,
             skillDir,
-            artifacts: skillArtifacts,
+            variantSpecs,
+            strictBaseline,
             model,
             judgeModel,
             outputDir,
@@ -226,4 +243,3 @@ export async function executeBatchEvaluationRuns({ skillDir, skillEntries, model
         await resolvedJobStore.save(job.jobId, job);
     return { report: batchReport, filePath };
 }
-//# sourceMappingURL=batch-evaluation-workflow.js.map

package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.d.ts

@@ -19,4 +19,3 @@ export declare function buildPowerWarnings(sampleCount: number, repeat: number,
 export declare function emitPowerWarnings(sampleCount: number, repeat: number, lang: Lang): void;
 export declare function buildIsolationWarnings(artifacts: Artifact[], strictBaseline: boolean | undefined): string[];
 export declare function emitIsolationWarnings(artifacts: Artifact[], strictBaseline: boolean | undefined): void;
-//# sourceMappingURL=preflight-warnings.d.ts.map

package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.js

@@ -64,4 +64,3 @@ export function emitIsolationWarnings(artifacts, strictBaseline) {
         process.stderr.write(`${w}\n`);
     }
 }
-//# sourceMappingURL=preflight-warnings.js.map

package/dist/eval-workflows/evaluation-pipeline/report-finalize.d.ts

@@ -29,4 +29,3 @@ export declare function finalizeEvaluationReport({ report, results, artifacts, v
     samples: Sample[];
 }): Report;
 export {};
-//# sourceMappingURL=report-finalize.d.ts.map

package/dist/eval-workflows/evaluation-pipeline/report-finalize.js

@@ -54,4 +54,3 @@ export function finalizeEvaluationReport({ report, results, artifacts, variantNa
     }
     return report;
 }
-//# sourceMappingURL=report-finalize.js.map

package/dist/eval-workflows/evaluation-pipeline/run-state.d.ts

@@ -57,4 +57,3 @@ export declare function finalizeSuccessfulRun(state: EvaluationRunState): {
 };
 export declare function persistSuccessfulJob(state: EvaluationRunState, job: EvaluationJob): Promise<void>;
 export declare function persistFailedJob(state: EvaluationRunState, err: unknown): Promise<void>;
-//# sourceMappingURL=run-state.d.ts.map

package/dist/eval-workflows/evaluation-pipeline/run-state.js

@@ -85,4 +85,3 @@ export async function persistFailedJob(state, err) {
         await state.resolvedJobStore.save(state.jobId, failedJob);
     }
 }
-//# sourceMappingURL=run-state.js.map

package/dist/eval-workflows/evaluation-pipeline/test-set-hash.d.ts

@@ -18,4 +18,3 @@
  */
 export declare function computeTestSetHash(samplesPath: string, sourceFiles?: string[]): string | null;
 export declare function _computeTestSetHashForTest(samplesPath: string, sourceFiles?: string[]): string | null;
-//# sourceMappingURL=test-set-hash.d.ts.map

package/dist/eval-workflows/evaluation-pipeline/test-set-hash.js

@@ -43,4 +43,3 @@ export function computeTestSetHash(samplesPath, sourceFiles) {
 export function _computeTestSetHashForTest(samplesPath, sourceFiles) {
     return computeTestSetHash(samplesPath, sourceFiles);
 }
-//# sourceMappingURL=test-set-hash.js.map

package/dist/eval-workflows/evaluation-pipeline.d.ts

@@ -94,4 +94,3 @@ export declare function executeEvaluationPipeline({ samplesPath, samplesBaseDir,
     report: Report;
     filePath: string | null;
 }>;
-//# sourceMappingURL=evaluation-pipeline.d.ts.map

package/dist/eval-workflows/evaluation-pipeline.js

@@ -176,4 +176,3 @@ requires: _requires, layeredStats = false, repeat, batch, judgeRepeat, judgeMode
         await stopAllServers();
     }
 }
-//# sourceMappingURL=evaluation-pipeline.js.map

package/dist/eval-workflows/evaluation-preparation.d.ts

@@ -12,17 +12,14 @@ export interface PreparedEvaluationRun {
     /** Sample bundle 内参与合并的所有源文件绝对路径(已排序)。test set hash 用它遍历。 */
     samplesSourceFiles: string[];
 }
-export declare function prepareEvaluationRun({ samplesPath, skillDir, variantSpecs, artifacts, dryRun, mcpConfig, strictBaseline, variantAllowedSkills, }: {
+export declare function prepareEvaluationRun({ samplesPath, skillDir, variantSpecs, dryRun, mcpConfig, strictBaseline, }: {
     samplesPath: string;
     skillDir: string;
     variantSpecs: VariantSpec[];
-    artifacts?: Artifact[];
     dryRun: boolean;
     mcpConfig?: string;
     /**  — default true, baseline-kind 自动 allowedSkills=[]。 */
     strictBaseline?: boolean;
-    /**  — eval.yaml 显式 allowedSkills per variant (overrides strictBaseline default). */
-    variantAllowedSkills?: Record<string, string[]>;
 }): Promise<PreparedEvaluationRun>;
 export declare function buildDryRunTaskReport({ model, judgeModels, executorName, samplesPath, skillDir, tasks, variantNames, }: {
     model: string;
@@ -57,4 +54,3 @@ export declare function buildDryRunTaskReport({ model, judgeModels, executorName
         hasSystem: boolean;
     }[];
 };
-//# sourceMappingURL=evaluation-preparation.d.ts.map

package/dist/eval-workflows/evaluation-preparation.js

@@ -5,22 +5,27 @@ import { resolveArtifacts } from '../inputs/skill-loader.js';
 import { buildVariantConfig } from '../eval-core/execution-strategy.js';
 import { loadMcpConfig, resolveMcpUrls } from '../inputs/mcp-resolver.js';
 import { resolveUrls } from '../inputs/url-fetcher.js';
-export async function prepareEvaluationRun({ samplesPath, skillDir, variantSpecs, artifacts, dryRun, mcpConfig, strictBaseline, variantAllowedSkills, }) {
+export async function prepareEvaluationRun({ samplesPath, skillDir, variantSpecs, dryRun, mcpConfig, strictBaseline, }) {
     const { samples, requires, baseDir: samplesBaseDir, sourceFiles: samplesSourceFiles } = loadSamples(samplesPath);
-    // Build expressions from specs (preserving order) and resolve to artifacts.
-    const variantExpressions = variantSpecs.map((spec) => spec.expr);
-    const resolvedArtifacts = artifacts || resolveArtifacts(resolve(skillDir), variantExpressions, { strictBaseline, variantAllowedSkills });
-    // Attach experimentRole to each artifact by matching spec.name to artifact.name.
-    const roleByName = {};
-    for (const spec of variantSpecs)
-        roleByName[spec.name] = spec.role;
-    for (const artifact of resolvedArtifacts) {
-        if (artifact.experimentRole)
-            continue; // preserve if already set (e.g. batch workflow)
-        const role = roleByName[artifact.name];
-        if (role)
-            artifact.experimentRole = role;
-    }
+    // 结构化传入 {expr, cwd}(与 variantSpecs 顺序一一对应),cwd 不再编码进 expr 字符串。
+    // resolveArtifacts 只保留 strictBaseline 默认(baseline → []);per-variant 隔离声明走
+    // spec.allowedSkills,在下面按 spec 身份绑定。
+    const variantInputs = variantSpecs.map((spec) => ({ expr: spec.expr, cwd: spec.cwd }));
+    const resolvedArtifacts = resolveArtifacts(resolve(skillDir), variantInputs, { strictBaseline });
+    // experimentRole / allowedSkills 按 spec 身份绑定。variantSpecs 与 resolvedArtifacts 顺序
+    // 一一对应(resolveArtifacts 每个 input 产出一个 artifact),按 index 绑定,并把 spec.name
+    // 同步成消歧后的最终唯一名。不能按 spec.name 匹配 artifact.name:同 basename 的 variant 被
+    // ensureUniqueVariantNames 消歧后(v1/greeter / v2/greeter)就和 spec 的派生短名(greeter)
+    // 对不上,会丢 role。
+    variantSpecs.forEach((spec, i) => {
+        const artifact = resolvedArtifacts[i];
+        artifact.experimentRole = spec.role;
+        // spec.allowedSkills 是隔离声明的单一来源(eval.yaml 经 configVariantsToSpecs、batch 经
+        // buildBatchVariantSpecs 都挂到 spec 上),显式声明优先于 strictBaseline 默认。
+        if (spec.allowedSkills !== undefined)
+            artifact.allowedSkills = spec.allowedSkills;
+        spec.name = artifact.name; // 同步唯一名,让 spec 与 report / variantNames 一致(放最后,前面还要用旧名查)
+    });
     if (!dryRun) {
         const mcpServers = loadMcpConfig(mcpConfig);
         const mcpResolved = mcpServers ? await resolveMcpUrls(samples, mcpServers) : new Set();
@@ -74,4 +79,3 @@ export function buildDryRunTaskReport({ model, judgeModels, executorName, sample
         }),
     };
 }
-//# sourceMappingURL=evaluation-preparation.js.map

package/dist/eval-workflows/messages.d.ts

@@ -1,4 +1,3 @@
 import type { Lang } from '../types/shared.js';
 export type EvalWorkflowMessageCode = 'power_warning_tiny_n' | 'power_warning_small_n' | 'power_warning_repeat_one' | 'doctor_gate_blocked';
 export declare function tEvalWorkflowMessage(code: EvalWorkflowMessageCode, lang?: Lang, params?: Record<string, string | number>): string;
-//# sourceMappingURL=messages.d.ts.map

package/dist/eval-workflows/messages.js

@@ -27,4 +27,3 @@ export function tEvalWorkflowMessage(code, lang = DEFAULT_LANG, params) {
     }
     return text;
 }
-//# sourceMappingURL=messages.js.map

package/dist/eval-workflows/run-evaluation.d.ts

@@ -63,7 +63,6 @@ export interface RunEvaluationOptions extends CommonEvaluationOptions {
     samplesPath: string;
     skillDir: string;
     variantSpecs?: VariantSpec[];
-    artifacts?: Artifact[];
     dryRun?: boolean;
     blind?: boolean;
     retry?: number;
@@ -71,12 +70,9 @@ export interface RunEvaluationOptions extends CommonEvaluationOptions {
     /** Explicit persisted run id. Used by batch workflows that need stable child ids. */
     runId?: string;
     /** strict-baseline default (CLI `--strict-baseline` default true).
-     *  When undefined, treated as true. baseline-kind variants get allowedSkills=[]
-     *  unless eval.yaml explicitly overrides per-variant. */
+     *  When undefined, treated as true. baseline-kind variants get allowedSkills=[].
+     *  per-variant 显式隔离声明走 variantSpecs[].allowedSkills(eval.yaml 经 configVariantsToSpecs)。 */
     strictBaseline?: boolean;
-    /** per-variant explicit allowedSkills override map (from eval.yaml).
-     *  Keyed by variant name. Always wins over strictBaseline default. */
-    variantAllowedSkills?: Record<string, string[]>;
 }
 export interface RunBatchEvaluationOptions extends CommonEvaluationOptions {
     skillDir: string;
@@ -122,7 +118,7 @@ export interface DryRunReport extends DryRunBase {
     samplesPath: string;
     tasks: DryRunTask[];
 }
-export declare function runEvaluation({ samplesPath, skillDir, variantSpecs, artifacts, model, outputDir, project, owner, tags, noJudge, dryRun, blind, concurrency, timeoutMs, noCache, executorName, jobStore, persistJob, onProgress, skipConnectivity, skipDoctor, lang, mcpConfig, verbose, retry, resume, runId, layeredStats, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, variantAllowedSkills, effort, noDiagnostic, }: RunEvaluationOptions): Promise<{
+export declare function runEvaluation({ samplesPath, skillDir, variantSpecs, model, outputDir, project, owner, tags, noJudge, dryRun, blind, concurrency, timeoutMs, noCache, executorName, jobStore, persistJob, onProgress, skipConnectivity, skipDoctor, lang, mcpConfig, verbose, retry, resume, runId, layeredStats, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, effort, noDiagnostic, }: RunEvaluationOptions): Promise<{
     report: Report | DryRunReport;
     filePath: string | null;
 }>;
@@ -137,7 +133,7 @@ export interface DryRunBatchReport extends DryRunBase {
     totalArtifacts: number;
     artifacts: DryRunBatchSkill[];
 }
-export declare function buildVarianceData(runs: Report[]): VarianceData | null;
+export declare function buildVarianceData(runs: Report[], bootstrapSamples?: number, seed?: number): VarianceData | null;
 export declare function runBatchEvaluation({ skillDir, model, outputDir, project, owner, tags, noJudge, dryRun, concurrency, timeoutMs, executorName, jobStore, persistJob, onProgress, onSkillProgress, skipConnectivity, skipDoctor, lang, mcpConfig, verbose, repeat, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, noCache, strictBaseline, variantAllowedSkills, }: RunBatchEvaluationOptions): Promise<{
     report: BatchEvaluationReport | DryRunBatchReport;
     filePath: string | null;
@@ -148,4 +144,3 @@ export declare function runMultiple({ repeat, onRepeatProgress, ...config }: Run
     filePath: string | null;
 }>;
 export {};
-//# sourceMappingURL=run-evaluation.d.ts.map

package/dist/eval-workflows/run-evaluation.js

@@ -1,14 +1,14 @@
 import { resolve } from 'node:path';
 import { DEFAULT_OUTPUT_DIR, persistReport } from '../eval-core/evaluation-reporting.js';
 import { createExecutor, DEFAULT_MODEL, JUDGE_MODEL } from '../executors/index.js';
-import { discoverBatchSkills, resolveArtifacts } from '../inputs/skill-loader.js';
+import { discoverBatchSkills } from '../inputs/skill-loader.js';
 import { confidenceInterval, tTest, effectSize } from '../eval-core/statistics.js';
-import { executeBatchEvaluationRuns } from './batch-evaluation-workflow.js';
+import { executeBatchEvaluationRuns, buildBatchVariantSpecs } from './batch-evaluation-workflow.js';
 import { buildDryRunTaskReport, prepareEvaluationRun, } from './evaluation-preparation.js';
 import { executeEvaluationPipeline } from './evaluation-pipeline.js';
 import { findSaturationPoint } from '../analysis/saturation.js';
-import { bootstrapMeanCI } from '../eval-core/bootstrap.js';
-export async function runEvaluation({ samplesPath, skillDir, variantSpecs = [], artifacts, model = DEFAULT_MODEL, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, noJudge = false, dryRun = false, blind = false, concurrency = 1, timeoutMs, noCache = false, executorName = 'claude', jobStore = null, persistJob = true, onProgress = null, skipConnectivity = false, skipDoctor = false, lang = 'zh', mcpConfig, verbose = false, retry = 0, resume, runId, layeredStats = false, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, variantAllowedSkills, effort, noDiagnostic, }) {
+import { bootstrapMeanCI, DEFAULT_BOOTSTRAP_ALPHA, DEFAULT_BOOTSTRAP_SAMPLES } from '../eval-core/bootstrap.js';
+export async function runEvaluation({ samplesPath, skillDir, variantSpecs = [], model = DEFAULT_MODEL, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, noJudge = false, dryRun = false, blind = false, concurrency = 1, timeoutMs, noCache = false, executorName = 'claude', jobStore = null, persistJob = true, onProgress = null, skipConnectivity = false, skipDoctor = false, lang = 'zh', mcpConfig, verbose = false, retry = 0, resume, runId, layeredStats = false, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, effort, noDiagnostic, }) {
     // Unified judgeModels → derive single-judge fields for downstream pipeline / grading
     // (which still operate on string `judgeModel` + `judgeExecutorName` fields per call).
     const effectiveJudgeModels = judgeModels && judgeModels.length > 0
@@ -20,11 +20,9 @@ export async function runEvaluation({ samplesPath, skillDir, variantSpecs = [],
         samplesPath,
         skillDir,
         variantSpecs,
-        artifacts,
         dryRun,
         mcpConfig,
         strictBaseline,
-        variantAllowedSkills,
     });
     // doctor 强制门禁: skill 静态结构 + 元数据 + 依赖 + 用例契约。
     // 在 dryRun 分支之前跑, 让 dry-run 也得到 doctor 覆盖(保护 garbage-in 的 verdict)。
@@ -99,7 +97,7 @@ export async function runEvaluation({ samplesPath, skillDir, variantSpecs = [],
         const { createFileStore } = await import('../server/report-store.js');
         const store = createFileStore(resolve(outputDir || DEFAULT_OUTPUT_DIR));
         const existing = await store.get(resume);
-        if (existing?.kind === 'evaluation') {
+        if (existing?.reportKind === 'evaluation') {
             existingResults = {};
             for (const entry of existing.results || []) {
                 existingResults[entry.sample_id] = entry.variants;
@@ -123,7 +121,7 @@ export async function runEvaluation({ samplesPath, skillDir, variantSpecs = [],
                     + `   新 entries 不隔离 → 与现有 entries 不可比。建议恢复默认 strict-baseline。\n`);
             }
         }
-        else if (existing?.kind === 'batch-evaluation') {
+        else if (existing?.reportKind === 'batch-evaluation') {
             process.stderr.write(`\n⚠️  report ${resume} is a BatchEvaluationReport; resume needs a child EvaluationReport, starting from scratch\n`);
         }
         else {
@@ -228,7 +226,7 @@ function buildComparisonMetric(scoresA, scoresB, meanA, meanB) {
  * verdict is computed — the user still sees the curve, just without the auto
  * "saturated at N=X" claim.
  */
-function buildSaturationData(runs) {
+function buildSaturationData(runs, bootstrapSamples = DEFAULT_BOOTSTRAP_SAMPLES, seed) {
     if (runs.length < 2)
         return undefined;
     const variants = runs[0].meta.variants ?? [];
@@ -259,7 +257,7 @@ function buildSaturationData(runs) {
                 cumulativeByVariant[variant] = [];
             cumulativeByVariant[variant].push([...acc[variant]]);
             // Per-checkpoint trace: bootstrap CI on cumulative scores.
-            const ci = bootstrapMeanCI(acc[variant], 0.05, 1000);
+            const ci = bootstrapMeanCI(acc[variant], DEFAULT_BOOTSTRAP_ALPHA, bootstrapSamples, seed);
             if (!tracesByVariant[variant])
                 tracesByVariant[variant] = [];
             tracesByVariant[variant].push({
@@ -276,7 +274,7 @@ function buildSaturationData(runs) {
             const cumulative = cumulativeByVariant[variant];
             if (!cumulative)
                 continue;
-            const r = findSaturationPoint(cumulative, 'bootstrap-ci-width');
+            const r = findSaturationPoint(cumulative, 'bootstrap-ci-width', undefined, undefined, bootstrapSamples, seed);
             verdicts[variant] = {
                 saturated: r.saturated,
                 atN: r.atN,
@@ -293,7 +291,7 @@ function buildSaturationData(runs) {
         ...(Object.keys(verdicts).length > 0 ? { verdicts } : {}),
     };
 }
-export function buildVarianceData(runs) {
+export function buildVarianceData(runs, bootstrapSamples = DEFAULT_BOOTSTRAP_SAMPLES, seed) {
     if (runs.length <= 1) {
         return null;
     }
@@ -358,7 +356,7 @@ export function buildVarianceData(runs) {
             });
         }
     }
-    const saturation = buildSaturationData(runs);
+    const saturation = buildSaturationData(runs, bootstrapSamples, seed);
     return {
         runs: runs.length,
         perVariant,
@@ -381,23 +379,17 @@ export async function runBatchEvaluation({ skillDir, model = DEFAULT_MODEL, outp
     if (dryRun) {
         // batch dry-run 走 per-entry runEvaluation(dryRun: true) — doctor 嵌入
         // 自动复用 runEvaluation 内部的 builder + 单一逻辑,不再在 batch 这边
-        // 旁路一套 doctor / 路径推断。entry artifact 拼装方式与 batch real-run
-        // (executeBatchEvaluationRuns) 严格一致, 避免 dry-run / real-run 漂移。
-        const skillDirAbs = resolve(skillDir);
+        // 旁路一套 doctor / 路径推断。entry 的 spec 构造(buildBatchVariantSpecs)与 batch
+        // real-run(executeBatchEvaluationRuns)同一处,避免 dry-run / real-run 漂移。
         const dryArtifacts = [];
         for (const entry of skillEntries) {
-            const skillArtifacts = resolveArtifacts(skillDirAbs, ['baseline', entry.skillPath], { strictBaseline, variantAllowedSkills }).map((artifact) => {
-                if (artifact.name === entry.skillPath) {
-                    return { ...artifact, name: entry.name, experimentRole: 'treatment' };
-                }
-                return { ...artifact, experimentRole: 'control' };
-            });
+            const variantSpecs = buildBatchVariantSpecs(entry, variantAllowedSkills);
             let entryReport;
             try {
                 const result = await runEvaluation({
                     samplesPath: entry.samplesPath,
                     skillDir,
-                    artifacts: skillArtifacts,
+                    variantSpecs,
                     model,
                     executorName,
                     dryRun: true,
@@ -406,7 +398,6 @@ export async function runBatchEvaluation({ skillDir, model = DEFAULT_MODEL, outp
                     lang,
                     mcpConfig,
                     strictBaseline,
-                    variantAllowedSkills,
                     // 透传 user 显式输入的参数,让 dry-run power / isolation warning 与
                     // 真实 run 一致(否则 batch dry-run 永远按 repeat=1 报警,误导用户)。
                     repeat,
@@ -516,7 +507,7 @@ export async function runMultiple({ repeat = 1, onRepeatProgress, ...config }) {
         runs.push(report);
     }
     const report = runs[runs.length - 1];
-    const aggregated = buildVarianceData(runs);
+    const aggregated = buildVarianceData(runs, config.bootstrapSamples ?? DEFAULT_BOOTSTRAP_SAMPLES);
     let filePath = null;
     if (aggregated) {
         report.variance = aggregated;
@@ -525,4 +516,3 @@ export async function runMultiple({ repeat = 1, onRepeatProgress, ...config }) {
     }
     return { report, aggregated, filePath };
 }
-//# sourceMappingURL=run-evaluation.js.map

package/dist/executors/anthropic-api.d.ts

@@ -1,3 +1,2 @@
 import type { ExecResult, ExecutorInput } from '../types/index.js';
 export declare function anthropicApiExecutor({ model, system, prompt, timeoutMs }: ExecutorInput): Promise<ExecResult>;
-//# sourceMappingURL=anthropic-api.d.ts.map

package/dist/executors/anthropic-api.js

@@ -5,7 +5,7 @@ export async function anthropicApiExecutor({ model, system, prompt, timeoutMs =
         throw new Error('ANTHROPIC_API_KEY environment variable is not set');
     const baseUrl = process.env.ANTHROPIC_BASE_URL || 'https://api.anthropic.com';
     const reqBody = {
-        model: model || 'claude-sonnet-4-5-20250514',
+        model: model || 'claude-sonnet-4-5',
         max_tokens: 8192,
         messages: [{ role: 'user', content: prompt }],
     };
@@ -40,4 +40,3 @@ export async function anthropicApiExecutor({ model, system, prompt, timeoutMs =
         return { ok: false, error, durationMs, durationApiMs: 0, inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheCreationTokens: 0, costUSD: 0, output: null, stopReason, numTurns: 0 };
     }
 }
-//# sourceMappingURL=anthropic-api.js.map

package/dist/executors/claude-cli.d.ts

@@ -1,3 +1,2 @@
 import type { ExecResult, ExecutorInput } from '../types/index.js';
 export declare function claudeCliExecutor({ model, system, prompt, cwd, skillDir, timeoutMs, allowedSkills, mocks, mocksBaseDir, mocksStrict, lean, effort }: ExecutorInput): Promise<ExecResult>;
-//# sourceMappingURL=claude-cli.d.ts.map

package/dist/executors/claude-cli.js

@@ -178,4 +178,3 @@ export async function claudeCliExecutor({ model, system, prompt, cwd, skillDir,
         mockHandle?.cleanup();
     }
 }
-//# sourceMappingURL=claude-cli.js.map

package/dist/executors/claude-sdk-trace.d.ts

@@ -7,4 +7,3 @@ export declare function extractAgentTrace(messages: ClaudeSdkBaseMessage[], time
     fullNumTurns: number;
     numSubAgents: number;
 };
-//# sourceMappingURL=claude-sdk-trace.d.ts.map

package/dist/executors/claude-sdk-trace.js

@@ -103,4 +103,3 @@ export function extractAgentTrace(messages, timestamps) {
     }
     return { turns, toolCalls, fullNumTurns, numSubAgents };
 }
-//# sourceMappingURL=claude-sdk-trace.js.map

package/dist/executors/claude-sdk.d.ts

@@ -13,4 +13,3 @@ export declare function buildSdkIsolationOptions(allowedSkills: string[] | undef
     disallowedTools?: string[];
 };
 export declare function claudeSdkExecutor({ model, system, prompt, cwd, skillDir, timeoutMs, verbose, allowedSkills, mocks, mocksBaseDir, mocksStrict, lean, effort }: ExecutorInput): Promise<ExecResult>;
-//# sourceMappingURL=claude-sdk.d.ts.map

package/dist/executors/claude-sdk.js

@@ -198,4 +198,3 @@ export async function claudeSdkExecutor({ model, system, prompt, cwd, skillDir,
         };
     }
 }
-//# sourceMappingURL=claude-sdk.js.map

package/dist/executors/codex-cli-trace.d.ts

@@ -7,4 +7,3 @@ export declare function extractCodexTrace(events: CodexEvent[]): {
     fullNumTurns: number;
     numSubAgents: number;
 };
-//# sourceMappingURL=codex-cli-trace.d.ts.map

package/dist/executors/codex-cli-trace.js

@@ -120,4 +120,3 @@ export function extractCodexTrace(events) {
     flushTurn();
     return { turns, toolCalls, fullNumTurns, numSubAgents };
 }
-//# sourceMappingURL=codex-cli-trace.js.map

package/dist/executors/codex-cli.d.ts

@@ -15,4 +15,3 @@ export declare function buildCodexArgs({ model, cwd, prompt }: {
     prompt: string;
 }): string[];
 export declare function codexCliExecutor({ model, system, prompt, cwd, skillDir, timeoutMs, allowedSkills, verbose }: ExecutorInput): Promise<ExecResult>;
-//# sourceMappingURL=codex-cli.d.ts.map

package/dist/executors/codex-cli.js

@@ -251,4 +251,3 @@ export async function codexCliExecutor({ model, system, prompt, cwd, skillDir, t
         };
     }
 }
-//# sourceMappingURL=codex-cli.js.map