oh-my-knowledge 0.52.3 → 0.54.0

package/dist/eval-core/evaluation-reporting.js

@@ -13,7 +13,7 @@ import { buildVariantConfig, resolveExecutionStrategy } from './execution-strate
 import { getJudgePromptHash } from '../grading/judge.js';
 import { getDiagnosticPromptHash, resolveDiagnosticTarget, } from '../grading/diagnostic.js';
 import { bootstrapMeanCI, bootstrapPairedDiffCI, DEFAULT_BOOTSTRAP_ALPHA, DEFAULT_BOOTSTRAP_SAMPLES, } from './bootstrap.js';
-import { getExecutorRuntimeFingerprint } from '../executors/runtime-fingerprint.js';
+import { resolveExecutorRuntimeFingerprint, } from '../executors/core/runtime-fingerprint.js';
 import { ownRecordValue, setOwnRecordValue, } from '../shared/record-count.js';
 import { writeJsonFileAtomic } from '../shared/atomic-json.js';
 import { hashSample } from './sample-fingerprint.js';
@@ -39,15 +39,25 @@ export { hashSample } from './sample-fingerprint.js';
 export function getCliVersion() {
     return PKG.version;
 }
-export function getGitInfo() {
-    // stdio 静默 stderr:在非 git 目录(如 omk init 出来的 demo)里 rev-parse 会打印
-    // `fatal: not a git repository` 到终端。catch 已把失败兜成 null(报告省略 git 信息),
-    // 这条 fatal 对用户是纯噪声,吞掉它。与 skill-loader 的 GIT_PROBE_STDIO 同口径。
+export function getGitInfo(cwd = process.cwd()) {
+    // porcelain v2 的 branch header 一次返回 commit / branch，后续状态行同时表达 dirty。
+    // 相比 rev-parse ×2 + status，少启动两个同步 Git 子进程；报告字段语义不变。
+    // stdio 静默 stderr：非 git 目录仍返回 null，不把 fatal 噪声泄漏给用户。
     const gitProbeStdio = ['ignore', 'pipe', 'ignore'];
     try {
-        const commit = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf-8', stdio: gitProbeStdio }).trim();
-        const branch = execFileSync('git', ['rev-parse', '--abbrev-ref', 'HEAD'], { encoding: 'utf-8', stdio: gitProbeStdio }).trim();
-        const dirty = execFileSync('git', ['status', '--porcelain'], { encoding: 'utf-8', stdio: gitProbeStdio }).trim().length > 0;
+        const output = execFileSync('git', ['status', '--porcelain=v2', '--branch'], {
+            cwd,
+            encoding: 'utf-8',
+            stdio: gitProbeStdio,
+        });
+        const lines = output.split('\n');
+        const oid = lines.find((line) => line.startsWith('# branch.oid '))?.slice('# branch.oid '.length).trim();
+        const head = lines.find((line) => line.startsWith('# branch.head '))?.slice('# branch.head '.length).trim();
+        if (!oid || oid === '(initial)' || !head)
+            return null;
+        const commit = oid;
+        const branch = head === '(detached)' ? 'HEAD' : head;
+        const dirty = lines.some((line) => line.length > 0 && !line.startsWith('# '));
         return { commit, commitShort: commit.slice(0, 7), branch, dirty };
     }
     catch {
@@ -64,15 +74,15 @@ function commonRuntime(runtimes) {
 function representativeRuntime(runtimes) {
     return Object.values(runtimes)[0];
 }
-export function buildExecutorRuntimesByVariant({ variants, model, executorName, tasks, artifacts, request, }) {
+export function buildExecutorRuntimesByVariant({ variants, model, executorName, tasks, artifacts, request, executor, }) {
     const runtimes = {};
     for (const task of tasks) {
         if (ownRecordValue(runtimes, task.variant))
             continue;
         const executionPlan = resolveExecutionStrategy(task, model, request?.timeoutMs, false);
-        setOwnRecordValue(runtimes, task.variant, getExecutorRuntimeFingerprint(executorName, model, {
+        setOwnRecordValue(runtimes, task.variant, resolveExecutorRuntimeFingerprint(executorName, model, {
             skillDir: executionPlan.input.skillDir,
-        }));
+        }, executor));
     }
     for (const variant of variants) {
         if (ownRecordValue(runtimes, variant))
@@ -86,13 +96,13 @@ export function buildExecutorRuntimesByVariant({ variants, model, executorName,
                 : artifact?.locator
                     ? dirname(artifact.locator)
                     : request?.skillDir);
-        setOwnRecordValue(runtimes, variant, getExecutorRuntimeFingerprint(executorName, model, {
+        setOwnRecordValue(runtimes, variant, resolveExecutorRuntimeFingerprint(executorName, model, {
             skillDir: fallbackSkillDir,
-        }));
+        }, executor));
     }
     return runtimes;
 }
-export function aggregateReport({ runId, variants, model, judgeModel, noJudge, executorName, samples, samplesBaseDir, tasks, results, totalCostUSD, artifacts, request, run, job, layeredStats, }) {
+export function aggregateReport({ runId, variants, model, judgeModel, noJudge, executorName, samples, samplesBaseDir, tasks, results, totalCostUSD, artifacts, request, run, job, layeredStats, executor, judgeExecutors, }) {
     const summary = {};
     for (const variant of variants) {
         const entries = Object.values(results)
@@ -172,10 +182,18 @@ export function aggregateReport({ runId, variants, model, judgeModel, noJudge, e
     const sampleHashes = Object.fromEntries(samples.map((sample) => [sample.sample_id, hashSample(sample, samplesBaseDir)]));
     const judgeRepeat = request?.judgeRepeat && request.judgeRepeat > 1 ? request.judgeRepeat : undefined;
     const runtimeOptions = { skillDir: request?.skillDir };
-    const executorRuntimes = buildExecutorRuntimesByVariant({ variants, model, executorName, tasks, artifacts, request });
+    const executorRuntimes = buildExecutorRuntimesByVariant({
+        variants,
+        model,
+        executorName,
+        tasks,
+        artifacts,
+        request,
+        executor,
+    });
     const executorRuntime = commonRuntime(executorRuntimes)
         ?? representativeRuntime(executorRuntimes)
-        ?? getExecutorRuntimeFingerprint(executorName, model, runtimeOptions);
+        ?? resolveExecutorRuntimeFingerprint(executorName, model, runtimeOptions, executor);
     // request.judgeModels is the authoritative source (always non-empty in new schema).
     // Fallback synthesizes a 1-entry from positional judgeModel/executorName for any
     // legacy caller not yet migrated to the array. noJudge ⇒ runtime undefined per entry.
@@ -183,7 +201,9 @@ export function aggregateReport({ runId, variants, model, judgeModel, noJudge, e
     const judgeModelsMeta = requestJudges.map((jc) => ({
         executor: jc.executor,
         model: jc.model,
-        ...(noJudge ? {} : { runtime: getExecutorRuntimeFingerprint(jc.executor, jc.model, runtimeOptions) }),
+        ...(noJudge ? {} : {
+            runtime: resolveExecutorRuntimeFingerprint(jc.executor, jc.model, runtimeOptions, judgeExecutors?.[jc.executor]),
+        }),
     }));
     const diagnosticEnabled = request?.noDiagnostic !== true;
     const diagnosticTarget = resolveDiagnosticTarget(requestJudges, executorName, model);
@@ -192,7 +212,8 @@ export function aggregateReport({ runId, variants, model, judgeModel, noJudge, e
             enabled: true,
             executor: diagnosticTarget.executor,
             model: diagnosticTarget.model,
-            runtime: getExecutorRuntimeFingerprint(diagnosticTarget.executor, diagnosticTarget.model),
+            runtime: resolveExecutorRuntimeFingerprint(diagnosticTarget.executor, diagnosticTarget.model, {}, judgeExecutors?.[diagnosticTarget.executor]
+                ?? (diagnosticTarget.executor === executorName ? executor : undefined)),
             promptHash: getDiagnosticPromptHash(),
         }
         : { enabled: false };

package/dist/eval-core/judge-independence.d.ts

@@ -1,5 +1,5 @@
 import type { Report } from '../types/index.js';
-import { type ExecutorVendor } from '../executors/shared.js';
+import { type ExecutorVendor } from '../executors/core/registry.js';
 /**
  * 评委独立性分析(单一来源,verdict caveat 与 analysis 诊断共用)。
  *

package/dist/eval-core/judge-independence.js

@@ -1,4 +1,4 @@
-import { executorVendor } from '../executors/shared.js';
+import { executorVendor } from '../executors/core/registry.js';
 export function analyzeJudgeIndependence(report) {
     const judges = report.meta?.judgeModels ?? [];
     const judgeVendors = judges.map((j) => executorVendor(j.executor));

package/dist/eval-core/report-document.js

@@ -71,6 +71,12 @@ function isRuntimeFingerprint(value) {
         || !['reported', 'not-reported', 'unknown'].includes(String(value.capabilities.costUSD))
         || !['native', 'best-effort', 'none', 'unknown'].includes(String(value.capabilities.trace))
         || !['full', 'full-no-partial', 'cwd-only', 'none', 'unknown'].includes(String(value.capabilities.skillIsolation))
+        || (value.auditability !== undefined
+            && (!isRecord(value.auditability)
+                || !['complete', 'partial'].includes(String(value.auditability.status))
+                || (value.auditability.reasons !== undefined
+                    && (!Array.isArray(value.auditability.reasons)
+                        || !value.auditability.reasons.every((reason) => typeof reason === 'string')))))
         || (value.sdk !== undefined && !isRuntimePackage(value.sdk)))
         return false;
     if (value.binary === undefined)

package/dist/eval-core/resume-compatibility.d.ts

@@ -17,6 +17,7 @@ export interface ResumeCompatibilityInput {
     samplesBaseDir?: string;
     tasks: Task[];
     artifacts: Artifact[];
+    executorOverrides?: Readonly<Record<string, import('../types/index.js').ExecutorFn>>;
 }
 export interface ResumeCompatibilityResult {
     compatible: boolean;

package/dist/eval-core/resume-compatibility.js

@@ -3,7 +3,7 @@ import { buildExecutorRuntimesByVariant, EVALUATION_REPORT_SCHEMA_VERSION, getCl
 import { hashSample } from './sample-fingerprint.js';
 import { getJudgePromptHash } from '../grading/judge.js';
 import { getDiagnosticPromptHash, resolveDiagnosticTarget, } from '../grading/diagnostic.js';
-import { getExecutorRuntimeFingerprint } from '../executors/runtime-fingerprint.js';
+import { resolveExecutorRuntimeFingerprint } from '../executors/core/runtime-fingerprint.js';
 function canonicalStringify(value) {
     if (value === undefined)
         return 'undefined';
@@ -73,6 +73,7 @@ export function checkResumeCompatibility(report, current) {
             skillDir: current.skillDir,
             timeoutMs: current.timeoutMs,
         },
+        executor: current.executorOverrides?.[current.executorName],
     });
     check('meta.executorRuntimes', runtimeFingerprints(report.meta.executorRuntimes), runtimeFingerprints(expectedExecutorRuntimes));
     const actualJudges = report.meta.judgeModels.map((judge) => ({
@@ -85,9 +86,9 @@ export function checkResumeCompatibility(report, current) {
         model: judge.model,
         ...(!current.noJudge
             ? {
-                runtime: getExecutorRuntimeFingerprint(judge.executor, judge.model, {
+                runtime: resolveExecutorRuntimeFingerprint(judge.executor, judge.model, {
                     skillDir: current.skillDir,
-                }).fingerprint,
+                }, current.executorOverrides?.[judge.executor]).fingerprint,
             }
             : {}),
     }));
@@ -102,7 +103,7 @@ export function checkResumeCompatibility(report, current) {
             enabled: true,
             executor: diagnosticTarget.executor,
             model: diagnosticTarget.model,
-            runtime: getExecutorRuntimeFingerprint(diagnosticTarget.executor, diagnosticTarget.model).fingerprint,
+            runtime: resolveExecutorRuntimeFingerprint(diagnosticTarget.executor, diagnosticTarget.model, {}, current.executorOverrides?.[diagnosticTarget.executor]).fingerprint,
             promptHash: getDiagnosticPromptHash(),
         }
         : { enabled: false };

package/dist/eval-workflows/batch-evaluation-workflow.js

@@ -1,7 +1,7 @@
 import { dirname } from 'node:path';
 import { DEFAULT_OUTPUT_DIR, EVALUATION_REPORT_SCHEMA_VERSION, generateRunId, getCliVersion, getGitInfo, persistReport, } from '../eval-core/evaluation-reporting.js';
 import { buildEvaluationRequest, createEvaluationRun, createSucceededJob, finalizeEvaluationRun } from '../eval-core/evaluation-job.js';
-import { getExecutorRuntimeFingerprint } from '../executors/runtime-fingerprint.js';
+import { getExecutorRuntimeFingerprint } from '../executors/core/runtime-fingerprint.js';
 import { createFileJobStore } from '../server/job-store.js';
 import { DEFAULT_JOBS_DIR } from '../eval-core/default-dirs.js';
 import { ownRecordValue, setOwnRecordValue } from '../shared/record-count.js';

package/dist/eval-workflows/evaluation-pipeline/preflight-warnings.js

@@ -16,6 +16,7 @@
 import { existsSync, readdirSync } from 'node:fs';
 import { homedir } from 'node:os';
 import { join, relative } from 'node:path';
+import { executorFamily } from '../../executors/core/registry.js';
 import { tEvalWorkflowMessage } from '../messages.js';
 export function buildPowerWarnings(sampleCount, repeat, lang = 'zh') {
     const warnings = [];
@@ -43,11 +44,12 @@ export function buildIsolationWarnings(artifacts, strictBaseline, options) {
     if (!hasUnisolatedBaseline)
         return [];
     const executorName = options.executorName;
+    const family = executorFamily(executorName);
     const home = options.homeDir ?? homedir();
     const cwd = options.cwd ?? process.cwd();
-    const roots = executorName === 'claude' || executorName === 'claude-sdk'
+    const roots = family === 'claude'
         ? [join(home, '.claude', 'skills'), join(cwd, '.claude', 'skills')]
-        : executorName === 'codex' || executorName === 'codex-sdk'
+        : family === 'codex'
             ? [
                 join(home, '.agents', 'skills'),
                 join(home, '.codex', 'skills'),

package/dist/eval-workflows/evaluation-pipeline.d.ts

@@ -43,6 +43,8 @@ export interface EvaluationPipelineOptions {
     judgeExecutorName: string;
     executor: ExecutorFn;
     judgeExecutor: ExecutorFn;
+    /** Embedding-host executors that must not be reconstructed from a CLI name. */
+    executorOverrides?: Readonly<Record<string, ExecutorFn>>;
     outputDir?: string | null;
     project?: string;
     owner?: string;
@@ -91,7 +93,7 @@ export interface EvaluationPipelineOptions {
     noDiagnostic?: boolean;
 }
 type VariantResult = import('../types/index.js').VariantResult;
-export declare function executeEvaluationPipeline({ samplesPath, samplesBaseDir, samplesSourceFiles, skillDir, samples, tasks, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, executor, judgeExecutor, outputDir, project, owner, tags, concurrency, timeoutMs, noCache, jobStore, persistJob, onProgress, skipConnectivity, verbose, retry, existingResults, requires: _requires, layeredStats, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, runId, lang, effort, noDiagnostic, }: EvaluationPipelineOptions): Promise<{
+export declare function executeEvaluationPipeline({ samplesPath, samplesBaseDir, samplesSourceFiles, skillDir, samples, tasks, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, executor, judgeExecutor, executorOverrides, outputDir, project, owner, tags, concurrency, timeoutMs, noCache, jobStore, persistJob, onProgress, skipConnectivity, verbose, retry, existingResults, requires: _requires, layeredStats, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, runId, lang, effort, noDiagnostic, }: EvaluationPipelineOptions): Promise<{
     report: Report;
     filePath: string | null;
 }>;

package/dist/eval-workflows/evaluation-pipeline.js

@@ -28,11 +28,11 @@ import { finalizeSuccessfulRun, initializeEvaluationRunState, persistFailedJob,
 import { finalizeEvaluationReport } from './evaluation-pipeline/report-finalize.js';
 import { emitIsolationWarnings, emitPowerWarnings } from './evaluation-pipeline/preflight-warnings.js';
 import { ownRecordValue, setOwnRecordValue } from '../shared/record-count.js';
-import { assertSamplesCompatibleWithExecutor } from '../executors/capabilities.js';
+import { assertSamplesCompatibleWithExecutor } from '../executors/core/capabilities.js';
 // 兼容 re-export:测试与 run-evaluation.ts 动态 import 仍打 evaluation-pipeline.js
 export { buildPowerWarnings, buildIsolationWarnings } from './evaluation-pipeline/preflight-warnings.js';
 export { _computeTestSetHashForTest } from './evaluation-pipeline/test-set-hash.js';
-export async function executeEvaluationPipeline({ samplesPath, samplesBaseDir, samplesSourceFiles, skillDir, samples, tasks, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, executor, judgeExecutor, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, concurrency = 1, timeoutMs, noCache = false, jobStore = null, persistJob = true, onProgress = null, skipConnectivity = false, verbose = false, retry = 0, existingResults, 
+export async function executeEvaluationPipeline({ samplesPath, samplesBaseDir, samplesSourceFiles, skillDir, samples, tasks, artifacts, model, judgeModel, noJudge, executorName, judgeExecutorName, executor, judgeExecutor, executorOverrides, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, concurrency = 1, timeoutMs, noCache = false, jobStore = null, persistJob = true, onProgress = null, skipConnectivity = false, verbose = false, retry = 0, existingResults, 
 // requires 现在由 runEvaluation 上游传给 doctor 处理; eval-pipeline 不再用
 // 但保留接口字段,避免破 programmatic API (类型层面接收, 内部忽略)
 requires: _requires, layeredStats = false, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias = true, budget, strictBaseline, runId, lang = 'zh', effort, noDiagnostic, }) {
@@ -87,7 +87,9 @@ requires: _requires, layeredStats = false, repeat, holdoutRatio, batch, judgeRep
             resolvedJudgeExecutors = Object.create(null);
             for (const jc of resolvedJudgeModels) {
                 if (!ownRecordValue(resolvedJudgeExecutors, jc.executor)) {
-                    setOwnRecordValue(resolvedJudgeExecutors, jc.executor, createExecutor(jc.executor));
+                    setOwnRecordValue(resolvedJudgeExecutors, jc.executor, executorOverrides?.[jc.executor]
+                        ?? (jc.executor === executorName ? executor : undefined)
+                        ?? createExecutor(jc.executor));
                 }
             }
         }
@@ -156,6 +158,8 @@ requires: _requires, layeredStats = false, repeat, holdoutRatio, batch, judgeRep
                 run,
                 job,
                 layeredStats,
+                executor,
+                judgeExecutors: resolvedJudgeExecutors,
             }),
             results,
             artifacts,

package/dist/eval-workflows/run-evaluation.d.ts

@@ -1,4 +1,4 @@
-import type { Artifact, BatchEvaluationReport, EvaluationReport, JobStore, ProgressCallback, Report, VarianceData, VariantSpec } from '../types/index.js';
+import type { Artifact, BatchEvaluationReport, EvaluationReport, ExecutorFn, JobStore, ProgressCallback, Report, VarianceData, VariantSpec } from '../types/index.js';
 export interface SkillProgressInfo {
     phase: string;
     skill: string;
@@ -17,6 +17,8 @@ interface CommonEvaluationOptions {
     timeoutMs?: number;
     /** Core API is runtime-neutral: CLI/default resolution happens before this boundary. */
     executorName: string;
+    /** Same-process executors supplied by an embedding host, keyed by executor name. */
+    executorOverrides?: Readonly<Record<string, ExecutorFn>>;
     jobStore?: JobStore | null;
     persistJob?: boolean;
     onProgress?: ProgressCallback | null;
@@ -122,7 +124,7 @@ export interface DryRunReport extends DryRunBase {
     samplesPath: string;
     tasks: DryRunTask[];
 }
-export declare function runEvaluation({ samplesPath, skillDir, variantSpecs, model, outputDir, project, owner, tags, noJudge, dryRun, concurrency, timeoutMs, noCache, executorName, jobStore, persistJob, onProgress, skipConnectivity, skipDoctor, lang, mcpConfig, verbose, retry, resume, runId, layeredStats, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, effort, noDiagnostic, }: RunEvaluationOptions): Promise<{
+export declare function runEvaluation({ samplesPath, skillDir, variantSpecs, model, outputDir, project, owner, tags, noJudge, dryRun, concurrency, timeoutMs, noCache, executorName, executorOverrides, jobStore, persistJob, onProgress, skipConnectivity, skipDoctor, lang, mcpConfig, verbose, retry, resume, runId, layeredStats, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, effort, noDiagnostic, }: RunEvaluationOptions): Promise<{
     report: Report | DryRunReport;
     filePath: string | null;
 }>;

package/dist/eval-workflows/run-evaluation.js

@@ -10,7 +10,7 @@ import { checkResumeCompatibility } from '../eval-core/resume-compatibility.js';
 import { findSaturationPoint } from '../analysis/saturation.js';
 import { bootstrapMeanCI, DEFAULT_BOOTSTRAP_ALPHA, DEFAULT_BOOTSTRAP_SAMPLES } from '../eval-core/bootstrap.js';
 import { ownRecordValue, setOwnRecordValue } from '../shared/record-count.js';
-export async function runEvaluation({ samplesPath, skillDir, variantSpecs = [], model, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, noJudge = false, dryRun = false, concurrency = 1, timeoutMs, noCache = false, executorName, jobStore = null, persistJob = true, onProgress = null, skipConnectivity = false, skipDoctor = false, lang = 'zh', mcpConfig, verbose = false, retry = 0, resume, runId, layeredStats = false, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, effort, noDiagnostic, }) {
+export async function runEvaluation({ samplesPath, skillDir, variantSpecs = [], model, outputDir = DEFAULT_OUTPUT_DIR, project, owner, tags, noJudge = false, dryRun = false, concurrency = 1, timeoutMs, noCache = false, executorName, executorOverrides, jobStore = null, persistJob = true, onProgress = null, skipConnectivity = false, skipDoctor = false, lang = 'zh', mcpConfig, verbose = false, retry = 0, resume, runId, layeredStats = false, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, effort, noDiagnostic, }) {
     // Unified judgeModels → derive single-judge fields for downstream pipeline / grading
     // (which still operate on string `judgeModel` + `judgeExecutorName` fields per call).
     const effectiveJudgeModels = judgeModels && judgeModels.length > 0
@@ -123,6 +123,7 @@ export async function runEvaluation({ samplesPath, skillDir, variantSpecs = [],
                 samplesBaseDir,
                 tasks,
                 artifacts: resolvedArtifacts,
+                executorOverrides,
             });
             if (compatibility.compatible) {
                 const sourceBySample = new Map(existing.results.map((entry) => [entry.sample_id, entry.variants]));
@@ -159,8 +160,10 @@ export async function runEvaluation({ samplesPath, skillDir, variantSpecs = [],
     const effectiveSkipConnectivity = existingResults !== undefined
         ? true
         : skipConnectivity;
-    const executor = createExecutor(executorName);
-    const judgeExecutor = createExecutor(judgeExecutorName || executorName);
+    const executor = executorOverrides?.[executorName] ?? createExecutor(executorName);
+    const effectiveJudgeExecutorName = judgeExecutorName || executorName;
+    const judgeExecutor = executorOverrides?.[effectiveJudgeExecutorName]
+        ?? createExecutor(effectiveJudgeExecutorName);
     return executeEvaluationPipeline({
         samplesPath,
         samplesBaseDir,
@@ -176,6 +179,7 @@ export async function runEvaluation({ samplesPath, skillDir, variantSpecs = [],
         judgeExecutorName: judgeExecutorName || executorName,
         executor,
         judgeExecutor,
+        executorOverrides,
         outputDir,
         project,
         owner,

package/dist/executors/{anthropic-api.d.ts → anthropic/api.d.ts}

@@ -1,2 +1,2 @@
-import type { ExecResult, ExecutorInput } from '../types/index.js';
+import type { ExecResult, ExecutorInput } from '../../types/index.js';
 export declare function anthropicApiExecutor({ model, system, prompt, timeoutMs }: ExecutorInput): Promise<ExecResult>;

package/dist/executors/{anthropic-api.js → anthropic/api.js}

@@ -1,5 +1,7 @@
-import { asErrorLike, DEFAULT_TIMEOUT_MS, errorMessage, readJsonResponse, responseBodyPreview, } from './shared.js';
-import { optionalTokenCount } from '../shared/token-usage.js';
+import { optionalTokenCount } from '../../shared/token-usage.js';
+import { DEFAULT_TIMEOUT_MS } from '../core/limits.js';
+import { readJsonResponse, responseBodyPreview } from '../core/http.js';
+import { asErrorLike, errorMessage } from '../core/runtime.js';
 export async function anthropicApiExecutor({ model, system, prompt, timeoutMs = DEFAULT_TIMEOUT_MS }) {
     const apiKey = process.env.ANTHROPIC_API_KEY;
     if (!apiKey)

package/dist/executors/{claude-cli.d.ts → anthropic/claude/cli.d.ts}

@@ -1,2 +1,2 @@
-import type { ExecResult, ExecutorInput } from '../types/index.js';
+import type { ExecResult, ExecutorInput } from '../../../types/index.js';
 export declare function claudeCliExecutor({ model, system, prompt, cwd, skillDir, timeoutMs, allowedSkills, mocks, mocksBaseDir, mocksStrict, lean, effort }: ExecutorInput): Promise<ExecResult>;

package/dist/executors/{claude-cli.js → anthropic/claude/cli.js}

@@ -1,6 +1,8 @@
-import { buildExecEnv, DEFAULT_TIMEOUT_MS, errorMessage, interruptedExecResult, MAX_BUFFER, spawnWithSigintPropagation, timeoutExecResult, } from './shared.js';
-import { materializeForCliConfigDir } from '../eval-core/mocks-runtime.js';
-import { buildClaudeResult, parseClaudeStreamJson } from './claude-protocol.js';
+import { materializeForCliConfigDir } from '../../../eval-core/mocks-runtime.js';
+import { buildClaudeResult, parseClaudeStreamJson } from './protocol.js';
+import { DEFAULT_TIMEOUT_MS, MAX_BUFFER } from '../../core/limits.js';
+import { buildExecEnv, errorMessage, interruptedExecResult, timeoutExecResult, } from '../../core/runtime.js';
+import { spawnWithSigintPropagation } from '../../core/subprocess.js';
 // claude CLI 用 `--disable-slash-commands` (文档:"Disable all skills") +
 // `--disallowedTools Skill` 实现与 SDK 等价的完全隔离。非空 skill 白名单不再支持
 // (它无法真正隔离,见 claude-sdk.ts buildSdkIsolationOptions),所以 [name1, ...] 必须 throw。

package/dist/executors/anthropic/claude/protocol.d.ts

@@ -0,0 +1,87 @@
+import type { ExecResult } from '../../../types/index.js';
+interface ClaudeTokenUsage {
+    input_tokens?: number;
+    output_tokens?: number;
+    cache_read_input_tokens?: number;
+    cache_creation_input_tokens?: number;
+}
+export interface ClaudeSdkQueryOptions {
+    model?: string;
+    systemPrompt?: string;
+    cwd: string;
+    permissionMode: 'bypassPermissions';
+    allowDangerouslySkipPermissions: true;
+    abortController: AbortController;
+    env: NodeJS.ProcessEnv;
+}
+export interface ClaudeSdkQueryInput {
+    prompt: string;
+    options: ClaudeSdkQueryOptions;
+}
+export interface ClaudeMessage {
+    type: string;
+    message?: {
+        role?: string;
+        content?: Array<{
+            type: string;
+            text?: string;
+            id?: string;
+            name?: string;
+            input?: unknown;
+        }>;
+    };
+    tool_use_id?: string;
+    content?: string | Array<{
+        type: string;
+        text?: string;
+    }>;
+    is_error?: boolean;
+}
+export interface ClaudeResultMessage extends ClaudeMessage {
+    type: 'result';
+    result?: string;
+    usage?: ClaudeTokenUsage;
+    total_cost_usd?: number;
+    duration_api_ms?: number;
+    duration_ms?: number;
+    num_turns?: number;
+    stop_reason?: string | null;
+    modelUsage?: Record<string, {
+        inputTokens?: number;
+        outputTokens?: number;
+        cacheReadInputTokens?: number;
+        cacheCreationInputTokens?: number;
+    }>;
+    subtype?: string;
+    errors?: string[];
+}
+export interface ClaudeSdkModule {
+    query: (opts: ClaudeSdkQueryInput) => AsyncIterable<ClaudeMessage>;
+}
+export interface ClaudeMeasurements {
+    durationMs: number;
+    durationApiMs: number;
+    inputTokens: number;
+    outputTokens: number;
+    cacheReadTokens: number;
+    cacheCreationTokens: number;
+    costUSD: number;
+    numTurns: number;
+}
+export declare function normalizeClaudeMeasurements(result: ClaudeResultMessage): ClaudeMeasurements | {
+    error: string;
+};
+export interface ClaudeStreamParseResult {
+    messages: ClaudeMessage[];
+    malformedLineCount: number;
+}
+export declare function parseClaudeStreamJson(stdout: string): ClaudeStreamParseResult;
+export declare function buildClaudeResult(options: {
+    messages: ClaudeMessage[];
+    wallClockDurationMs: number;
+    source: 'claude stream-json' | 'claude-sdk';
+    malformedLineCount?: number;
+    forcedError?: string;
+    messageTimestamps?: number[];
+}): ExecResult;
+export {};

package/dist/executors/{claude-protocol.js → anthropic/claude/protocol.js}

@@ -1,6 +1,6 @@
-import { checkedSumTokenCounts, nonNegativeMetric, optionalTokenCount, } from '../shared/token-usage.js';
-import { extractAgentTrace, isClaudeSdkResultMessage } from './claude-sdk-trace.js';
-export function normalizeClaudeSdkMeasurements(result) {
+import { checkedSumTokenCounts, nonNegativeMetric, optionalTokenCount, } from '../../../shared/token-usage.js';
+import { extractClaudeTrace, isClaudeResultMessage } from './trace.js';
+export function normalizeClaudeMeasurements(result) {
     const durationMs = optionalTokenCount(result.duration_ms);
     const durationApiMs = optionalTokenCount(result.duration_api_ms);
     const numTurns = optionalTokenCount(result.num_turns);
@@ -96,8 +96,8 @@ export function parseClaudeStreamJson(stdout) {
 }
 export function buildClaudeResult(options) {
     const { messages, wallClockDurationMs, source, malformedLineCount = 0, forcedError, messageTimestamps, } = options;
-    const resultMessages = messages.filter(isClaudeSdkResultMessage);
-    const trace = extractAgentTrace(messages, messageTimestamps);
+    const resultMessages = messages.filter(isClaudeResultMessage);
+    const trace = extractClaudeTrace(messages, messageTimestamps);
     const traceFields = {
         fullNumTurns: trace.fullNumTurns,
         numSubAgents: trace.numSubAgents,
@@ -133,7 +133,7 @@ export function buildClaudeResult(options) {
         };
     }
     const result = resultMessages[0];
-    const measurements = normalizeClaudeSdkMeasurements(result);
+    const measurements = normalizeClaudeMeasurements(result);
     if ('error' in measurements) {
         errors.push(measurements.error);
         return {

package/dist/executors/{claude-sdk.d.ts → anthropic/claude/sdk.d.ts}

@@ -1,5 +1,5 @@
-import type { ExecResult, ExecutorInput } from '../types/index.js';
-export { normalizeClaudeSdkMeasurements } from './claude-protocol.js';
+import type { ExecResult, ExecutorInput } from '../../../types/index.js';
+export { normalizeClaudeMeasurements } from './protocol.js';
 /**
  * Map ExecutorInput.allowedSkills to SDK query options for skill isolation.
  *   undefined → {} (SDK default: full ~/.claude/skills/ discovery)

package/dist/executors/{claude-sdk.js → anthropic/claude/sdk.js}

@@ -1,11 +1,14 @@
 import { join } from 'node:path';
 import { mkdirSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
-import { asErrorLike, buildExecEnv, DEFAULT_TIMEOUT_MS, errorMessage, interruptedExecResult, registerSigintSubscriber, timeoutExecResult, } from './shared.js';
-import { buildSdkHookCallback } from '../eval-core/mocks-runtime.js';
-import { buildClaudeResult } from './claude-protocol.js';
-export { normalizeClaudeSdkMeasurements } from './claude-protocol.js';
+import { DEFAULT_TIMEOUT_MS } from '../../core/limits.js';
+import { asErrorLike, buildExecEnv, errorMessage, interruptedExecResult, timeoutExecResult, } from '../../core/runtime.js';
+import { registerSigintSubscriber } from '../../core/subprocess.js';
+import { buildSdkHookCallback } from '../../../eval-core/mocks-runtime.js';
+import { buildClaudeResult } from './protocol.js';
+export { normalizeClaudeMeasurements } from './protocol.js';
 let sdkQuery = null;
+const CLAUDE_AGENT_SDK_PACKAGE = '@anthropic-ai/claude-agent-sdk';
 /**
  * Map ExecutorInput.allowedSkills to SDK query options for skill isolation.
  *   undefined → {} (SDK default: full ~/.claude/skills/ discovery)
@@ -28,7 +31,7 @@ export function buildSdkIsolationOptions(allowedSkills) {
 }
 async function getSdkQuery() {
     if (!sdkQuery) {
-        const sdk = await import('@anthropic-ai/claude-agent-sdk');
+        const sdk = await import(CLAUDE_AGENT_SDK_PACKAGE);
         sdkQuery = sdk.query;
     }
     return sdkQuery;

package/dist/executors/anthropic/claude/trace.d.ts

@@ -0,0 +1,9 @@
+import type { ToolCallInfo, TurnInfo } from '../../../types/index.js';
+import type { ClaudeMessage } from './protocol.js';
+export declare function isClaudeResultMessage(message: ClaudeMessage): boolean;
+export declare function extractClaudeTrace(messages: ClaudeMessage[], timestamps?: number[]): {
+    turns: TurnInfo[];
+    toolCalls: ToolCallInfo[];
+    fullNumTurns: number;
+    numSubAgents: number;
+};

package/dist/executors/{claude-sdk-trace.js → anthropic/claude/trace.js}

@@ -1,10 +1,10 @@
-import { safeSliceForJson } from '../util/safe-slice.js';
-import { isToolResultFailureText } from '../observability/text-signals.js';
-import { normalizeToolIdentity } from '../shared/tool-identity.js';
-export function isClaudeSdkResultMessage(message) {
+import { safeSliceForJson } from '../../../util/safe-slice.js';
+import { isToolResultFailureText } from '../../../observability/text-signals.js';
+import { normalizeToolIdentity } from '../../../shared/tool-identity.js';
+export function isClaudeResultMessage(message) {
     return message.type === 'result';
 }
-export function extractAgentTrace(messages, timestamps) {
+export function extractClaudeTrace(messages, timestamps) {
     const turns = [];
     const toolCalls = [];
     const pendingToolUse = new Map();

package/dist/executors/{capabilities.d.ts → core/capabilities.d.ts}

@@ -1,8 +1,6 @@
-import type { ExecutorFn, ExecutorInput, Sample } from '../types/index.js';
-export type SampleMockSupport = 'native-hooks' | 'delegated-script' | 'unsupported';
-export interface ExecutorCapabilities {
-    sampleMocks: SampleMockSupport;
-}
+import type { ExecutorFn, ExecutorInput, Sample } from '../../types/index.js';
+import { type ExecutorCapabilities } from './registry.js';
+export type { ExecutorCapabilities, SampleMockSupport } from './registry.js';
 /**
  * Custom script executors receive the OMK_MOCK_* protocol environment and own
  * the final adapter. Built-ins are explicit so unsupported runtimes can never

package/dist/executors/{capabilities.js → core/capabilities.js}

@@ -1,20 +1,13 @@
-const BUILTIN_CAPABILITIES = {
-    claude: { sampleMocks: 'native-hooks' },
-    'claude-sdk': { sampleMocks: 'native-hooks' },
-    codex: { sampleMocks: 'unsupported' },
-    'codex-sdk': { sampleMocks: 'unsupported' },
-    gemini: { sampleMocks: 'unsupported' },
-    'anthropic-api': { sampleMocks: 'unsupported' },
-    'openai-api': { sampleMocks: 'unsupported' },
-};
+import { getExecutorDescriptor, } from './registry.js';
 /**
  * Custom script executors receive the OMK_MOCK_* protocol environment and own
  * the final adapter. Built-ins are explicit so unsupported runtimes can never
  * silently turn mock assertions into model failures.
  */
 export function getExecutorCapabilities(executorName) {
-    return BUILTIN_CAPABILITIES[executorName]
-        ?? { sampleMocks: 'delegated-script' };
+    return {
+        sampleMocks: getExecutorDescriptor(executorName)?.sampleMocks ?? 'delegated-script',
+    };
 }
 export function executorSupportsSampleMocks(executorName) {
     return getExecutorCapabilities(executorName).sampleMocks !== 'unsupported';

package/dist/executors/core/http.d.ts

@@ -0,0 +1,6 @@
+export interface JsonResponseBody<T> {
+    data: T | null;
+    rawBody: string;
+}
+export declare function readJsonResponse<T>(response: Response): Promise<JsonResponseBody<T>>;
+export declare function responseBodyPreview(rawBody: string, maxLength?: number): string;