oh-my-knowledge 0.52.3 → 0.53.0

package/dist/dsh-plugin/index.js

@@ -0,0 +1,156 @@
+import { dirname, join, resolve } from 'node:path';
+import { computeVerdict } from '../eval-core/verdict.js';
+import { configVariantsToSpecs, loadEvalConfig } from '../inputs/eval-config.js';
+import { runEvaluation, runMultiple } from '../eval-workflows/run-evaluation.js';
+import { createDshHostExecutor, } from './host-executor.js';
+export const name = 'omk-dsh-plugin';
+export const inject = ['agentPresets', 'agents', 'commands', 'tools'];
+const USAGE = '用法：/omk eval <eval.yaml>';
+function commandPath(rawInput) {
+    const match = /^\s*eval\s+(.+?)\s*$/u.exec(rawInput);
+    if (!match)
+        return undefined;
+    const value = match[1]?.trim();
+    if (!value)
+        return undefined;
+    if ((value.startsWith('"') && value.endsWith('"'))
+        || (value.startsWith("'") && value.endsWith("'"))) {
+        return value.slice(1, -1);
+    }
+    return value;
+}
+function projectRoot(configPath) {
+    return dirname(configPath);
+}
+function modelFor(agent, configured) {
+    const model = configured?.trim() || agent.options.model?.trim();
+    if (!model) {
+        throw new Error('当前 DSH session 没有可继承的模型，且 eval.yaml 未配置 model。');
+    }
+    return model;
+}
+function normalizeJudgeModels(configured, model) {
+    if (!configured || configured.length === 0)
+        return [{ executor: 'dsh-host', model }];
+    return configured.map((judge) => {
+        if (judge.executor === 'dsh-host') {
+            throw new Error('dsh-host 是 OMK 内部执行器标识；评委要复用当前 DSH 时，请使用 executor: dsh 或省略 judgeModels。');
+        }
+        return {
+            ...judge,
+            executor: judge.executor === 'dsh' ? 'dsh-host' : judge.executor,
+        };
+    });
+}
+async function executeEvalCommand(ctx, invocation, requestedPath) {
+    const cwd = invocation.agent.session.header.cwd ?? process.cwd();
+    const configPath = resolve(cwd, requestedPath);
+    const config = loadEvalConfig(configPath);
+    if (config.executor) {
+        return {
+            kind: 'error',
+            text: 'DSH 内运行 OMK 时，被测执行器固定为当前 DSH；请删除 eval.yaml 中的顶层 executor。',
+        };
+    }
+    if (config.effort) {
+        return {
+            kind: 'error',
+            text: 'DSH 的 reasoning effort 是 provider-owned 枚举，当前无法与 OMK 五档无损映射；请删除 eval.yaml 中的 effort，并在 DSH profile 中固定模型推理配置。',
+        };
+    }
+    const model = modelFor(invocation.agent, config.model);
+    const executor = createDshHostExecutor(ctx, {
+        parentAgent: invocation.agent,
+        signal: invocation.signal,
+    });
+    const judgeModels = normalizeJudgeModels(config.judgeModels, model);
+    const currentSessionProvesConnectivity = config.model === undefined
+        && ((config.noJudge ?? false) || judgeModels.every((judge) => (judge.executor === 'dsh-host' && judge.model === model)));
+    const root = projectRoot(configPath);
+    const outputDir = join(root, '.omk', 'reports');
+    const options = {
+        samplesPath: config.samples,
+        skillDir: join(root, 'skills'),
+        variantSpecs: configVariantsToSpecs(config.variants),
+        model,
+        outputDir,
+        noJudge: config.noJudge ?? false,
+        concurrency: config.concurrency ?? 1,
+        timeoutMs: config.timeoutMs,
+        noCache: config.noCache ?? false,
+        executorName: 'dsh-host',
+        executorOverrides: { 'dsh-host': executor },
+        judgeModels,
+        skipConnectivity: currentSessionProvesConnectivity,
+        skipDoctor: config.skipDoctor ?? false,
+        lang: 'zh',
+        mcpConfig: config.mcpConfig,
+        bootstrap: config.bootstrap ?? true,
+        bootstrapSamples: config.bootstrapSamples,
+        holdoutRatio: config.holdoutRatio,
+        judgeRepeat: config.judgeRepeat,
+        lengthDebias: config.lengthDebias,
+        budget: config.budget,
+        strictBaseline: config.strictBaseline,
+        noDiagnostic: config.noDiagnostic,
+    };
+    const result = config.repeat && config.repeat > 1
+        ? await runMultiple({ ...options, repeat: config.repeat })
+        : await runEvaluation(options);
+    const report = result.report;
+    const goldMessages = [];
+    if (config.goldDir) {
+        const { attachGoldAgreementToReport } = await import('../grading/gold-cli.js');
+        const gold = attachGoldAgreementToReport({
+            report,
+            goldDir: config.goldDir,
+            outputDir,
+            samples: config.bootstrapSamples,
+        });
+        if (gold.result && gold.gold) {
+            const alpha = Number.isFinite(gold.result.agreement.alpha)
+                ? gold.result.agreement.alpha.toFixed(3)
+                : '不可计算';
+            goldMessages.push(`Gold 一致性：Krippendorff α=${alpha}，N=${gold.result.agreement.sampleCount}，标注者=${gold.gold.metadata.annotator}`);
+            if (gold.result.contaminationWarning) {
+                goldMessages.push(`⚠ Gold 污染提示：${gold.result.contaminationWarning}`);
+            }
+        }
+        else {
+            goldMessages.push(`⚠ Gold 数据未加载：${gold.loadIssues.join('；') || config.goldDir}`);
+        }
+    }
+    const verdict = computeVerdict(report);
+    return {
+        kind: 'success',
+        text: [
+            `OMK 评测完成：${verdict.level}`,
+            verdict.headline,
+            `报告：${report.id}`,
+            ...(result.filePath ? [`文件：${result.filePath}`] : []),
+            ...goldMessages,
+        ].join('\n'),
+    };
+}
+/** Register `/omk eval <eval.yaml>` in every DSH command-capable surface. */
+export function apply(ctx) {
+    ctx.commands.register({
+        name: 'omk',
+        description: '在当前 DeepSeek Harness 中运行 OMK 对照评测',
+        input: { hint: 'eval <eval.yaml>' },
+        async handler(invocation) {
+            const path = commandPath(invocation.rawInput);
+            if (!path)
+                return { kind: 'error', text: USAGE };
+            try {
+                return await executeEvalCommand(ctx, invocation, path);
+            }
+            catch (error) {
+                return {
+                    kind: 'error',
+                    text: error instanceof Error ? error.message : String(error),
+                };
+            }
+        },
+    });
+}

package/dist/dsh-plugin/protocol.d.ts

@@ -0,0 +1,21 @@
+import type { ExecResult } from '../types/index.js';
+type UnknownRecord = Record<string, unknown>;
+/** Host-owned DSH events consumed by OMK without importing DSH runtime types. */
+export interface DshHostRunResult {
+    rootSessionId: string;
+    finalResponse: string;
+    /** Root and descendant events in host-observed receive order. */
+    events: Array<{
+        sessionId: string;
+        event: UnknownRecord;
+        traceRole: 'main' | 'subagent';
+    }>;
+    childSessionIds: string[];
+}
+/**
+ * Map DSH's append-only session log into OMK's source-neutral executor result.
+ * Root and descendant token/tool evidence are included; final output remains
+ * the root session's last assistant text.
+ */
+export declare function buildDshHostResult(result: DshHostRunResult, wallClockDurationMs: number): ExecResult;
+export {};

package/dist/dsh-plugin/protocol.js

@@ -0,0 +1,229 @@
+import { checkedSumTokenCounts, optionalTokenCount } from '../shared/token-usage.js';
+import { normalizeToolIdentity } from '../shared/tool-identity.js';
+import { safeSliceForJson } from '../util/safe-slice.js';
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+function nonEmptyString(value) {
+    return typeof value === 'string' && value.trim() ? value : undefined;
+}
+function finiteInteger(value) {
+    return typeof value === 'number' && Number.isSafeInteger(value) && value >= 0
+        ? value
+        : undefined;
+}
+function eventTimestamp(event) {
+    const time = finiteInteger(event.time);
+    if (time === undefined)
+        return undefined;
+    try {
+        return new Date(time).toISOString();
+    }
+    catch {
+        return undefined;
+    }
+}
+function contentBlocks(value) {
+    return Array.isArray(value) ? value.filter(isRecord) : [];
+}
+function textFromBlocks(value) {
+    if (typeof value === 'string')
+        return value;
+    return contentBlocks(value)
+        .flatMap((block) => block.type === 'text' && typeof block.text === 'string'
+        ? [block.text]
+        : [])
+        .join('');
+}
+function serializableToolInput(value) {
+    if (typeof value !== 'string')
+        return value ?? null;
+    try {
+        return JSON.parse(value);
+    }
+    catch {
+        return value;
+    }
+}
+function boundedToolOutput(value) {
+    if (typeof value === 'string')
+        return safeSliceForJson(value, 50_000);
+    return value ?? null;
+}
+function resultBlock(data) {
+    const message = isRecord(data.message) ? data.message : undefined;
+    return contentBlocks(message?.content).find((block) => block.type === 'tool-result');
+}
+function collectEvents(result) {
+    return result.events.map(({ sessionId, event, traceRole }) => ({
+        sessionId,
+        event,
+        traceRole,
+    }));
+}
+function terminalReason(events, rootSessionId) {
+    const terminal = [...events].reverse().find(({ sessionId, event }) => (sessionId === rootSessionId && event.type === 'turn/end'));
+    if (!terminal || !isRecord(terminal.event.data)) {
+        return { stopReason: 'unknown', error: 'dsh runtime reached idle without a root turn/end event' };
+    }
+    const reason = isRecord(terminal.event.data.reason) ? terminal.event.data.reason : undefined;
+    const stopReason = nonEmptyString(reason?.kind) ?? 'unknown';
+    if (stopReason === 'error') {
+        const failure = isRecord(reason?.error) ? reason.error : undefined;
+        return {
+            stopReason,
+            error: nonEmptyString(failure?.message) ?? 'dsh turn failed',
+        };
+    }
+    if (['aborted', 'blocked', 'interrupted'].includes(stopReason)) {
+        return { stopReason, error: `dsh turn ended with ${stopReason}` };
+    }
+    if (stopReason === 'max-tokens') {
+        return { stopReason, error: 'dsh turn ended after reaching the output-token limit' };
+    }
+    return { stopReason };
+}
+/**
+ * Map DSH's append-only session log into OMK's source-neutral executor result.
+ * Root and descendant token/tool evidence are included; final output remains
+ * the root session's last assistant text.
+ */
+export function buildDshHostResult(result, wallClockDurationMs) {
+    const events = collectEvents(result);
+    const tools = new Map();
+    const orderedTools = [];
+    const turns = [];
+    let inputTokens = 0;
+    let outputTokens = 0;
+    let cacheReadTokens = 0;
+    let cacheCreationTokens = 0;
+    let usageReported = true;
+    let assistantMessages = 0;
+    let rootTurns = 0;
+    for (const record of events) {
+        const { event, sessionId, traceRole } = record;
+        const data = isRecord(event.data) ? event.data : {};
+        const eventType = nonEmptyString(event.type);
+        if (eventType === 'turn/end' && sessionId === result.rootSessionId)
+            rootTurns += 1;
+        if (eventType === 'assistant/message') {
+            assistantMessages += 1;
+            const message = isRecord(data.message) ? data.message : {};
+            const text = textFromBlocks(message.content);
+            const usage = isRecord(data.usage) ? data.usage : undefined;
+            const nextInput = optionalTokenCount(usage?.inputTokens);
+            const nextOutput = optionalTokenCount(usage?.outputTokens);
+            const nextCacheRead = usage?.cacheReadTokens === undefined
+                ? 0
+                : optionalTokenCount(usage.cacheReadTokens);
+            const nextCacheWrite = usage?.cacheWriteTokens === undefined
+                ? 0
+                : optionalTokenCount(usage.cacheWriteTokens);
+            const sums = usage && nextInput !== undefined && nextOutput !== undefined
+                && nextCacheRead !== undefined && nextCacheWrite !== undefined
+                ? {
+                    input: checkedSumTokenCounts(inputTokens, nextInput),
+                    output: checkedSumTokenCounts(outputTokens, nextOutput),
+                    cacheRead: checkedSumTokenCounts(cacheReadTokens, nextCacheRead),
+                    cacheWrite: checkedSumTokenCounts(cacheCreationTokens, nextCacheWrite),
+                }
+                : undefined;
+            if (!sums || Object.values(sums).some((value) => value === undefined)) {
+                usageReported = false;
+            }
+            else {
+                inputTokens = sums.input ?? inputTokens;
+                outputTokens = sums.output ?? outputTokens;
+                cacheReadTokens = sums.cacheRead ?? cacheReadTokens;
+                cacheCreationTokens = sums.cacheWrite ?? cacheCreationTokens;
+            }
+            turns.push({
+                role: 'assistant',
+                content: text,
+            });
+            continue;
+        }
+        if (eventType === 'user/message') {
+            const message = isRecord(data.message) ? data.message : data;
+            turns.push({ role: 'user', content: textFromBlocks(message.content) });
+            continue;
+        }
+        if (eventType === 'tool/call') {
+            const callId = nonEmptyString(data.callId);
+            const sourceName = nonEmptyString(data.name);
+            if (!callId || !sourceName)
+                continue;
+            const identity = normalizeToolIdentity({ sourceName });
+            const timestamp = eventTimestamp(event);
+            const info = {
+                tool: identity.name,
+                ...(identity.sourceName && { sourceTool: identity.sourceName }),
+                ...(identity.namespace && { toolNamespace: identity.namespace }),
+                ...(identity.provider && { toolProvider: identity.provider }),
+                input: serializableToolInput(data.arguments),
+                output: null,
+                status: 'unknown',
+                statusSource: 'unknown',
+                success: false,
+                callInstanceId: `${sessionId}:${callId}`,
+                toolUseId: callId,
+                ...(timestamp && { timestamp }),
+                sourceTrace: `dsh-host:${sessionId}`,
+                traceRole,
+            };
+            const turn = {
+                role: 'tool',
+                content: 'null',
+                toolCalls: [info],
+            };
+            const mutable = { info, turn };
+            tools.set(`${sessionId}\0${callId}`, mutable);
+            orderedTools.push(mutable);
+            turns.push(turn);
+            continue;
+        }
+        if (eventType === 'tool/result') {
+            const block = resultBlock(data);
+            const callId = nonEmptyString(block?.toolCallId);
+            if (!callId)
+                continue;
+            const tool = tools.get(`${sessionId}\0${callId}`);
+            if (!tool)
+                continue;
+            const failed = Boolean(data.error) || block?.isError === true;
+            const resultContent = contentBlocks(block?.content);
+            const textOutput = textFromBlocks(resultContent);
+            tool.info.output = boundedToolOutput(textOutput || resultContent);
+            tool.info.status = failed ? 'failure' : 'success';
+            tool.info.statusSource = 'runtime';
+            tool.info.success = !failed;
+            tool.turn.content = typeof tool.info.output === 'string'
+                ? tool.info.output
+                : JSON.stringify(tool.info.output);
+        }
+    }
+    const terminal = terminalReason(events, result.rootSessionId);
+    const output = result.finalResponse || null;
+    const successfulStop = terminal.stopReason === 'completed';
+    const error = terminal.error ?? (!output ? 'dsh runtime produced no root assistant output' : undefined);
+    return {
+        ok: successfulStop && Boolean(output) && !error,
+        output,
+        durationMs: wallClockDurationMs,
+        durationApiMs: wallClockDurationMs,
+        inputTokens,
+        outputTokens,
+        cacheReadTokens,
+        cacheCreationTokens,
+        tokenUsageReportedByExecutor: usageReported && assistantMessages > 0,
+        costUSD: 0,
+        costReportedByExecutor: false,
+        stopReason: terminal.stopReason,
+        numTurns: rootTurns,
+        fullNumTurns: assistantMessages,
+        numSubAgents: new Set(result.childSessionIds).size,
+        ...(error && { error }),
+        turns,
+        toolCalls: orderedTools.map(({ info }) => info),
+    };
+}

package/dist/eval-core/comparability.js

@@ -44,6 +44,9 @@ function packageReasons(pkg, label) {
 }
 function runtimeUnverifiableReasons(runtime) {
     const reasons = [];
+    if (runtime.auditability?.status === 'partial') {
+        reasons.push(...(runtime.auditability.reasons ?? ['runtime auditability partial']));
+    }
     if (runtime.runtimeKind === 'api')
         return reasons;
     if (runtime.runtimeKind === 'agent-sdk') {

package/dist/eval-core/evaluation-execution.js

@@ -5,22 +5,14 @@ import { grade } from '../grading/index.js';
 import { checkFacts } from './fact-checker.js';
 import { resolveExecutionStrategy } from './execution-strategy.js';
 import { DEFAULT_CACHE_DIR, DEFAULT_ISOLATED_CWD_DIR } from './default-dirs.js';
-import { getExecutorRuntimeFingerprint } from '../executors/runtime-fingerprint.js';
+import { resolveExecutorRuntimeFingerprint } from '../executors/core/runtime-fingerprint.js';
+import { isRegisteredExecutorName } from '../executors/core/registry.js';
 import { ownRecordValue, setOwnRecordValue, } from '../shared/record-count.js';
 import { executorResultValidationError, normalizeExecResultToolIdentities, } from '../shared/executor-result.js';
 import { hashSampleExecutionDependencies } from './sample-fingerprint.js';
 import { resolveDiagnosticTarget } from '../grading/diagnostic.js';
 import { dirname, join, resolve } from 'node:path';
 import { mkdir, mkdtemp, rm } from 'node:fs/promises';
-const PREFLIGHT_RUNTIME_LABEL_EXECUTORS = new Set([
-    'claude',
-    'claude-sdk',
-    'codex',
-    'codex-sdk',
-    'gemini',
-    'anthropic-api',
-    'openai-api',
-]);
 async function runWithConcurrency(tasks, concurrency, fn) {
     let index = 0;
     async function worker() {
@@ -119,9 +111,9 @@ export async function executeTasks({ tasks, executor, executorName, model, noJud
         const total = tasks.length;
         onProgress?.({ phase: 'start', completed: idx, total, sample_id: task.sample_id, variant: task.variant });
         const executionPlan = resolveExecutionStrategy(task, model, timeoutMs, verbose, effort, samplesBaseDir);
-        const executorRuntime = getExecutorRuntimeFingerprint(effectiveExecutorName, model, {
+        const executorRuntime = resolveExecutorRuntimeFingerprint(effectiveExecutorName, model, {
             skillDir: executionPlan.input.skillDir,
-        });
+        }, executor);
         let execResult;
         // include allowedSkills in cache key so isolation-on / isolation-off runs
         // don't share cache entries, and include runtime fingerprint so a binary/SDK bump
@@ -390,7 +382,7 @@ export async function executeTasks({ tasks, executor, executorName, model, noJud
     return { results, totalCostUSD, skipped, budgetExhausted };
 }
 export function preflightRuntimeLabel(executorName, model) {
-    return PREFLIGHT_RUNTIME_LABEL_EXECUTORS.has(executorName) ? `${executorName}:${model}` : `custom:${model}`;
+    return isRegisteredExecutorName(executorName) ? `${executorName}:${model}` : `custom:${model}`;
 }
 export async function preflight(executor, model, timeoutMs = 180000, label) {
     const result = await executor({

package/dist/eval-core/evaluation-reporting.d.ts

@@ -1,18 +1,19 @@
-import { getExecutorRuntimeFingerprint } from '../executors/runtime-fingerprint.js';
+import { getExecutorRuntimeFingerprint } from '../executors/core/runtime-fingerprint.js';
 import type { Artifact, Report, Sample, Task, VariantResult, GitInfo, EvaluationJob, EvaluationRequest, EvaluationRun, ReportDocument } from '../types/index.js';
 export declare const DEFAULT_OUTPUT_DIR: string;
 export declare const EVALUATION_REPORT_SCHEMA_VERSION = 5;
 export declare function hashString(str: string): string;
 export { hashSample } from './sample-fingerprint.js';
 export declare function getCliVersion(): string;
-export declare function getGitInfo(): GitInfo | null;
-export declare function buildExecutorRuntimesByVariant({ variants, model, executorName, tasks, artifacts, request, }: {
+export declare function getGitInfo(cwd?: string): GitInfo | null;
+export declare function buildExecutorRuntimesByVariant({ variants, model, executorName, tasks, artifacts, request, executor, }: {
     variants: string[];
     model: string;
     executorName: string;
     tasks: Task[];
     artifacts: Artifact[];
     request?: Pick<EvaluationRequest, 'skillDir' | 'timeoutMs'>;
+    executor?: import('../types/index.js').ExecutorFn;
 }): Record<string, ReturnType<typeof getExecutorRuntimeFingerprint>>;
 interface AggregateReportOptions {
     runId: string;
@@ -31,8 +32,10 @@ interface AggregateReportOptions {
     run?: EvaluationRun;
     job?: EvaluationJob;
     layeredStats?: boolean;
+    executor?: import('../types/index.js').ExecutorFn;
+    judgeExecutors?: Readonly<Record<string, import('../types/index.js').ExecutorFn>>;
 }
-export declare function aggregateReport({ runId, variants, model, judgeModel, noJudge, executorName, samples, samplesBaseDir, tasks, results, totalCostUSD, artifacts, request, run, job, layeredStats, }: AggregateReportOptions): Report;
+export declare function aggregateReport({ runId, variants, model, judgeModel, noJudge, executorName, samples, samplesBaseDir, tasks, results, totalCostUSD, artifacts, request, run, job, layeredStats, executor, judgeExecutors, }: AggregateReportOptions): Report;
 export type PersistableReport = ReportDocument;
 export declare function persistReport(report: PersistableReport, outputDir: string | null): string | null;
 /**

package/dist/eval-core/evaluation-reporting.js

@@ -13,7 +13,7 @@ import { buildVariantConfig, resolveExecutionStrategy } from './execution-strate
 import { getJudgePromptHash } from '../grading/judge.js';
 import { getDiagnosticPromptHash, resolveDiagnosticTarget, } from '../grading/diagnostic.js';
 import { bootstrapMeanCI, bootstrapPairedDiffCI, DEFAULT_BOOTSTRAP_ALPHA, DEFAULT_BOOTSTRAP_SAMPLES, } from './bootstrap.js';
-import { getExecutorRuntimeFingerprint } from '../executors/runtime-fingerprint.js';
+import { resolveExecutorRuntimeFingerprint, } from '../executors/core/runtime-fingerprint.js';
 import { ownRecordValue, setOwnRecordValue, } from '../shared/record-count.js';
 import { writeJsonFileAtomic } from '../shared/atomic-json.js';
 import { hashSample } from './sample-fingerprint.js';
@@ -39,15 +39,25 @@ export { hashSample } from './sample-fingerprint.js';
 export function getCliVersion() {
     return PKG.version;
 }
-export function getGitInfo() {
-    // stdio 静默 stderr:在非 git 目录(如 omk init 出来的 demo)里 rev-parse 会打印
-    // `fatal: not a git repository` 到终端。catch 已把失败兜成 null(报告省略 git 信息),
-    // 这条 fatal 对用户是纯噪声,吞掉它。与 skill-loader 的 GIT_PROBE_STDIO 同口径。
+export function getGitInfo(cwd = process.cwd()) {
+    // porcelain v2 的 branch header 一次返回 commit / branch，后续状态行同时表达 dirty。
+    // 相比 rev-parse ×2 + status，少启动两个同步 Git 子进程；报告字段语义不变。
+    // stdio 静默 stderr：非 git 目录仍返回 null，不把 fatal 噪声泄漏给用户。
     const gitProbeStdio = ['ignore', 'pipe', 'ignore'];
     try {
-        const commit = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf-8', stdio: gitProbeStdio }).trim();
-        const branch = execFileSync('git', ['rev-parse', '--abbrev-ref', 'HEAD'], { encoding: 'utf-8', stdio: gitProbeStdio }).trim();
-        const dirty = execFileSync('git', ['status', '--porcelain'], { encoding: 'utf-8', stdio: gitProbeStdio }).trim().length > 0;
+        const output = execFileSync('git', ['status', '--porcelain=v2', '--branch'], {
+            cwd,
+            encoding: 'utf-8',
+            stdio: gitProbeStdio,
+        });
+        const lines = output.split('\n');
+        const oid = lines.find((line) => line.startsWith('# branch.oid '))?.slice('# branch.oid '.length).trim();
+        const head = lines.find((line) => line.startsWith('# branch.head '))?.slice('# branch.head '.length).trim();
+        if (!oid || oid === '(initial)' || !head)
+            return null;
+        const commit = oid;
+        const branch = head === '(detached)' ? 'HEAD' : head;
+        const dirty = lines.some((line) => line.length > 0 && !line.startsWith('# '));
         return { commit, commitShort: commit.slice(0, 7), branch, dirty };
     }
     catch {
@@ -64,15 +74,15 @@ function commonRuntime(runtimes) {
 function representativeRuntime(runtimes) {
     return Object.values(runtimes)[0];
 }
-export function buildExecutorRuntimesByVariant({ variants, model, executorName, tasks, artifacts, request, }) {
+export function buildExecutorRuntimesByVariant({ variants, model, executorName, tasks, artifacts, request, executor, }) {
     const runtimes = {};
     for (const task of tasks) {
         if (ownRecordValue(runtimes, task.variant))
             continue;
         const executionPlan = resolveExecutionStrategy(task, model, request?.timeoutMs, false);
-        setOwnRecordValue(runtimes, task.variant, getExecutorRuntimeFingerprint(executorName, model, {
+        setOwnRecordValue(runtimes, task.variant, resolveExecutorRuntimeFingerprint(executorName, model, {
             skillDir: executionPlan.input.skillDir,
-        }));
+        }, executor));
     }
     for (const variant of variants) {
         if (ownRecordValue(runtimes, variant))
@@ -86,13 +96,13 @@ export function buildExecutorRuntimesByVariant({ variants, model, executorName,
                 : artifact?.locator
                     ? dirname(artifact.locator)
                     : request?.skillDir);
-        setOwnRecordValue(runtimes, variant, getExecutorRuntimeFingerprint(executorName, model, {
+        setOwnRecordValue(runtimes, variant, resolveExecutorRuntimeFingerprint(executorName, model, {
             skillDir: fallbackSkillDir,
-        }));
+        }, executor));
     }
     return runtimes;
 }
-export function aggregateReport({ runId, variants, model, judgeModel, noJudge, executorName, samples, samplesBaseDir, tasks, results, totalCostUSD, artifacts, request, run, job, layeredStats, }) {
+export function aggregateReport({ runId, variants, model, judgeModel, noJudge, executorName, samples, samplesBaseDir, tasks, results, totalCostUSD, artifacts, request, run, job, layeredStats, executor, judgeExecutors, }) {
     const summary = {};
     for (const variant of variants) {
         const entries = Object.values(results)
@@ -172,10 +182,18 @@ export function aggregateReport({ runId, variants, model, judgeModel, noJudge, e
     const sampleHashes = Object.fromEntries(samples.map((sample) => [sample.sample_id, hashSample(sample, samplesBaseDir)]));
     const judgeRepeat = request?.judgeRepeat && request.judgeRepeat > 1 ? request.judgeRepeat : undefined;
     const runtimeOptions = { skillDir: request?.skillDir };
-    const executorRuntimes = buildExecutorRuntimesByVariant({ variants, model, executorName, tasks, artifacts, request });
+    const executorRuntimes = buildExecutorRuntimesByVariant({
+        variants,
+        model,
+        executorName,
+        tasks,
+        artifacts,
+        request,
+        executor,
+    });
     const executorRuntime = commonRuntime(executorRuntimes)
         ?? representativeRuntime(executorRuntimes)
-        ?? getExecutorRuntimeFingerprint(executorName, model, runtimeOptions);
+        ?? resolveExecutorRuntimeFingerprint(executorName, model, runtimeOptions, executor);
     // request.judgeModels is the authoritative source (always non-empty in new schema).
     // Fallback synthesizes a 1-entry from positional judgeModel/executorName for any
     // legacy caller not yet migrated to the array. noJudge ⇒ runtime undefined per entry.
@@ -183,7 +201,9 @@ export function aggregateReport({ runId, variants, model, judgeModel, noJudge, e
     const judgeModelsMeta = requestJudges.map((jc) => ({
         executor: jc.executor,
         model: jc.model,
-        ...(noJudge ? {} : { runtime: getExecutorRuntimeFingerprint(jc.executor, jc.model, runtimeOptions) }),
+        ...(noJudge ? {} : {
+            runtime: resolveExecutorRuntimeFingerprint(jc.executor, jc.model, runtimeOptions, judgeExecutors?.[jc.executor]),
+        }),
     }));
     const diagnosticEnabled = request?.noDiagnostic !== true;
     const diagnosticTarget = resolveDiagnosticTarget(requestJudges, executorName, model);
@@ -192,7 +212,8 @@ export function aggregateReport({ runId, variants, model, judgeModel, noJudge, e
             enabled: true,
             executor: diagnosticTarget.executor,
             model: diagnosticTarget.model,
-            runtime: getExecutorRuntimeFingerprint(diagnosticTarget.executor, diagnosticTarget.model),
+            runtime: resolveExecutorRuntimeFingerprint(diagnosticTarget.executor, diagnosticTarget.model, {}, judgeExecutors?.[diagnosticTarget.executor]
+                ?? (diagnosticTarget.executor === executorName ? executor : undefined)),
             promptHash: getDiagnosticPromptHash(),
         }
         : { enabled: false };

package/dist/eval-core/judge-independence.d.ts

@@ -1,5 +1,5 @@
 import type { Report } from '../types/index.js';
-import { type ExecutorVendor } from '../executors/shared.js';
+import { type ExecutorVendor } from '../executors/core/registry.js';
 /**
  * 评委独立性分析(单一来源,verdict caveat 与 analysis 诊断共用)。
  *

package/dist/eval-core/judge-independence.js

@@ -1,4 +1,4 @@
-import { executorVendor } from '../executors/shared.js';
+import { executorVendor } from '../executors/core/registry.js';
 export function analyzeJudgeIndependence(report) {
     const judges = report.meta?.judgeModels ?? [];
     const judgeVendors = judges.map((j) => executorVendor(j.executor));

package/dist/eval-core/report-document.js

@@ -71,6 +71,12 @@ function isRuntimeFingerprint(value) {
         || !['reported', 'not-reported', 'unknown'].includes(String(value.capabilities.costUSD))
         || !['native', 'best-effort', 'none', 'unknown'].includes(String(value.capabilities.trace))
         || !['full', 'full-no-partial', 'cwd-only', 'none', 'unknown'].includes(String(value.capabilities.skillIsolation))
+        || (value.auditability !== undefined
+            && (!isRecord(value.auditability)
+                || !['complete', 'partial'].includes(String(value.auditability.status))
+                || (value.auditability.reasons !== undefined
+                    && (!Array.isArray(value.auditability.reasons)
+                        || !value.auditability.reasons.every((reason) => typeof reason === 'string')))))
         || (value.sdk !== undefined && !isRuntimePackage(value.sdk)))
         return false;
     if (value.binary === undefined)

package/dist/eval-core/resume-compatibility.d.ts

@@ -17,6 +17,7 @@ export interface ResumeCompatibilityInput {
     samplesBaseDir?: string;
     tasks: Task[];
     artifacts: Artifact[];
+    executorOverrides?: Readonly<Record<string, import('../types/index.js').ExecutorFn>>;
 }
 export interface ResumeCompatibilityResult {
     compatible: boolean;