oh-my-knowledge 0.48.0 → 0.49.0

package/dist/eval-core/resume-compatibility.js

@@ -0,0 +1,141 @@
+import { buildVariantConfig } from './execution-strategy.js';
+import { buildExecutorRuntimesByVariant, EVALUATION_REPORT_SCHEMA_VERSION, getCliVersion, } from './evaluation-reporting.js';
+import { hashSample } from './sample-fingerprint.js';
+import { getJudgePromptHash } from '../grading/judge.js';
+import { getDiagnosticPromptHash, resolveDiagnosticTarget, } from '../grading/diagnostic.js';
+import { getExecutorRuntimeFingerprint } from '../executors/runtime-fingerprint.js';
+function canonicalStringify(value) {
+    if (value === undefined)
+        return 'undefined';
+    if (value === null || typeof value !== 'object')
+        return JSON.stringify(value) ?? 'null';
+    if (Array.isArray(value))
+        return `[${value.map(canonicalStringify).join(',')}]`;
+    const record = value;
+    const keys = Object.keys(record)
+        .filter((key) => record[key] !== undefined)
+        .sort();
+    return `{${keys.map((key) => `${JSON.stringify(key)}:${canonicalStringify(record[key])}`).join(',')}}`;
+}
+function same(left, right) {
+    return canonicalStringify(left) === canonicalStringify(right);
+}
+function runtimeFingerprints(runtimes) {
+    if (!runtimes)
+        return undefined;
+    return Object.fromEntries(Object.entries(runtimes).map(([variant, runtime]) => [variant, runtime.fingerprint]));
+}
+/**
+ * Decide whether successful entries from an earlier report can be inserted
+ * into the current measurement without mixing protocols. Every value compared
+ * here is either the report's public measurement evidence or a request option
+ * that can alter a persisted VariantResult.
+ */
+export function checkResumeCompatibility(report, current) {
+    const mismatches = [];
+    const check = (field, actual, expected) => {
+        if (!same(actual, expected))
+            mismatches.push(field);
+    };
+    check('meta.schemaVersion', report.meta.schemaVersion, EVALUATION_REPORT_SCHEMA_VERSION);
+    check('meta.cliVersion', report.meta.cliVersion, getCliVersion());
+    check('meta.nodeVersion', report.meta.nodeVersion, process.version);
+    check('meta.variants', report.meta.variants, current.variants);
+    check('meta.model', report.meta.model, current.model);
+    check('meta.executor', report.meta.executor, current.executorName);
+    check('meta.effort', report.meta.effort, current.effort);
+    check('meta.noJudge', report.meta.noJudge === true, current.noJudge);
+    check('meta.judgeRepeat', report.meta.judgeRepeat ?? 1, current.judgeRepeat ?? 1);
+    check('meta.sampleCount', report.meta.sampleCount, current.samples.length);
+    check('meta.taskCount', report.meta.taskCount, current.tasks.length);
+    const expectedArtifactHashes = Object.fromEntries(current.artifacts.map((artifact) => [
+        artifact.name,
+        artifact.contentHash ?? 'no-skill',
+    ]));
+    check('meta.artifactHashes', report.meta.artifactHashes, expectedArtifactHashes);
+    const expectedSampleHashes = Object.fromEntries(current.samples.map((sample) => [
+        sample.sample_id,
+        hashSample(sample, current.samplesBaseDir),
+    ]));
+    check('meta.sampleHashes', report.meta.sampleHashes, expectedSampleHashes);
+    check('meta.variantConfigs', report.meta.variantConfigs, current.artifacts.map(buildVariantConfig));
+    check('meta.skillIsolation', report.meta.skillIsolation, Object.fromEntries(current.artifacts.map((artifact) => [
+        artifact.name,
+        artifact.allowedSkills ?? null,
+    ])));
+    const expectedExecutorRuntimes = buildExecutorRuntimesByVariant({
+        variants: current.variants,
+        model: current.model,
+        executorName: current.executorName,
+        tasks: current.tasks,
+        artifacts: current.artifacts,
+        request: {
+            skillDir: current.skillDir,
+            timeoutMs: current.timeoutMs,
+        },
+    });
+    check('meta.executorRuntimes', runtimeFingerprints(report.meta.executorRuntimes), runtimeFingerprints(expectedExecutorRuntimes));
+    const actualJudges = report.meta.judgeModels.map((judge) => ({
+        executor: judge.executor,
+        model: judge.model,
+        runtime: judge.runtime?.fingerprint,
+    }));
+    const expectedJudges = current.judgeModels.map((judge) => ({
+        executor: judge.executor,
+        model: judge.model,
+        ...(!current.noJudge
+            ? {
+                runtime: getExecutorRuntimeFingerprint(judge.executor, judge.model, {
+                    skillDir: current.skillDir,
+                }).fingerprint,
+            }
+            : {}),
+    }));
+    check('meta.judgeModels', actualJudges, expectedJudges);
+    if (!current.noJudge) {
+        check('meta.judgePromptHash', report.meta.judgePromptHash, getJudgePromptHash(current.lengthDebias !== false));
+    }
+    const diagnosticEnabled = current.noDiagnostic !== true;
+    const diagnosticTarget = resolveDiagnosticTarget(current.judgeModels, current.executorName, current.model);
+    const expectedDiagnostic = diagnosticEnabled
+        ? {
+            enabled: true,
+            executor: diagnosticTarget.executor,
+            model: diagnosticTarget.model,
+            runtime: getExecutorRuntimeFingerprint(diagnosticTarget.executor, diagnosticTarget.model).fingerprint,
+            promptHash: getDiagnosticPromptHash(),
+        }
+        : { enabled: false };
+    const actualDiagnostic = report.meta.diagnostic
+        ? {
+            enabled: report.meta.diagnostic.enabled,
+            ...(report.meta.diagnostic.executor
+                ? { executor: report.meta.diagnostic.executor }
+                : {}),
+            ...(report.meta.diagnostic.model ? { model: report.meta.diagnostic.model } : {}),
+            ...(report.meta.diagnostic.runtime
+                ? { runtime: report.meta.diagnostic.runtime.fingerprint }
+                : {}),
+            ...(report.meta.diagnostic.promptHash
+                ? { promptHash: report.meta.diagnostic.promptHash }
+                : {}),
+        }
+        : undefined;
+    check('meta.diagnostic', actualDiagnostic, expectedDiagnostic);
+    const request = report.meta.request;
+    if (!request) {
+        mismatches.push('meta.request');
+    }
+    else {
+        check('meta.request.timeoutMs', request.timeoutMs, current.timeoutMs);
+        check('meta.request.budget', request.budget, current.budget);
+        check('meta.request.retry', request.retry ?? 0, current.retry ?? 0);
+        check('meta.request.noDiagnostic', request.noDiagnostic === true, current.noDiagnostic === true);
+    }
+    if (report.meta.budgetExhausted)
+        mismatches.push('meta.budgetExhausted');
+    return {
+        compatible: mismatches.length === 0,
+        mismatches: [...new Set(mismatches)],
+    };
+}

package/dist/eval-core/sample-fingerprint.d.ts

@@ -0,0 +1,12 @@
+import type { Sample } from '../types/index.js';
+/**
+ * Content fingerprint for files that can change executor output while the
+ * inline sample contract remains byte-for-byte identical.
+ */
+export declare function hashSampleExecutionDependencies(sample: Sample, baseDir?: string): string | undefined;
+/**
+ * Stable hash of the complete sample measurement contract, including external
+ * mock fixtures and the statically resolvable ESM import graph of custom
+ * assertion modules when a sample base directory is available.
+ */
+export declare function hashSample(sample: Sample, baseDir?: string): string;

package/dist/eval-core/sample-fingerprint.js

@@ -0,0 +1,193 @@
+import { createHash } from 'node:crypto';
+import { existsSync, readFileSync, statSync } from 'node:fs';
+import { builtinModules, createRequire } from 'node:module';
+import { dirname, isAbsolute, relative, resolve, } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { initSync, parse } from 'es-module-lexer';
+const MAX_ASSERTION_MODULE_FILES = 512;
+const MAX_ASSERTION_MODULE_BYTES = 32 * 1024 * 1024;
+const BUILTIN_MODULES = new Set([
+    ...builtinModules,
+    ...builtinModules.map((name) => `node:${name}`),
+]);
+initSync();
+function canonicalStringify(value) {
+    if (value === null || typeof value !== 'object')
+        return JSON.stringify(value) ?? 'null';
+    if (Array.isArray(value)) {
+        return `[${value.map((entry) => entry === undefined ? 'null' : canonicalStringify(entry)).join(',')}]`;
+    }
+    const record = value;
+    const entries = Object.keys(record)
+        .filter((key) => record[key] !== undefined)
+        .sort();
+    return `{${entries.map((key) => `${JSON.stringify(key)}:${canonicalStringify(record[key])}`).join(',')}}`;
+}
+function hashString(value) {
+    return createHash('sha256').update(value).digest('hex');
+}
+function dependencyContentFingerprint(path, baseDir) {
+    if (!baseDir && !isAbsolute(path))
+        return 'base-dir-unavailable';
+    const resolvedPath = isAbsolute(path) ? path : resolve(baseDir, path);
+    try {
+        return `sha256:${hashString(readFileSync(resolvedPath))}`;
+    }
+    catch (error) {
+        const code = error.code;
+        return `unavailable:${code ?? 'unknown'}`;
+    }
+}
+function customAssertionPaths(assertions) {
+    if (!assertions)
+        return [];
+    return assertions.flatMap((assertion) => {
+        const record = assertion;
+        const current = assertion.type === 'custom' && typeof record.fn === 'string'
+            ? [record.fn]
+            : [];
+        const children = assertion.type === 'assert-set' && Array.isArray(assertion.children)
+            ? customAssertionPaths(assertion.children)
+            : [];
+        return [...current, ...children];
+    });
+}
+function displayDependencyPath(path, baseDir) {
+    if (!baseDir)
+        return path;
+    const rel = relative(baseDir, path);
+    return rel || '.';
+}
+function resolveImportedModule(specifier, importer) {
+    if (BUILTIN_MODULES.has(specifier))
+        return undefined;
+    if (specifier.startsWith('file:')) {
+        try {
+            return fileURLToPath(specifier);
+        }
+        catch {
+            return undefined;
+        }
+    }
+    if (specifier.startsWith('.') || specifier.startsWith('/')) {
+        const clean = specifier.replace(/[?#].*$/, '');
+        const path = specifier.startsWith('/') ? clean : resolve(dirname(importer), clean);
+        return existsSync(path) && statSync(path).isFile() ? path : undefined;
+    }
+    try {
+        const resolved = createRequire(importer).resolve(specifier);
+        return BUILTIN_MODULES.has(resolved) ? undefined : resolved;
+    }
+    catch {
+        return undefined;
+    }
+}
+function customAssertionModuleEntries(path, baseDir) {
+    if (!baseDir && !isAbsolute(path)) {
+        return [[`module:${path}`, 'base-dir-unavailable']];
+    }
+    const entryPath = isAbsolute(path) ? path : resolve(baseDir, path);
+    const entries = [];
+    const visited = new Set();
+    let totalBytes = 0;
+    const visit = (modulePath) => {
+        if (visited.has(modulePath))
+            return;
+        if (visited.size >= MAX_ASSERTION_MODULE_FILES) {
+            throw new Error(`custom assertion module graph exceeds ${MAX_ASSERTION_MODULE_FILES} files: ${entryPath}`);
+        }
+        visited.add(modulePath);
+        let source;
+        try {
+            source = readFileSync(modulePath);
+        }
+        catch (error) {
+            const code = error.code;
+            entries.push([
+                `module:${displayDependencyPath(modulePath, baseDir)}`,
+                `unavailable:${code ?? 'unknown'}`,
+            ]);
+            return;
+        }
+        totalBytes += source.byteLength;
+        if (totalBytes > MAX_ASSERTION_MODULE_BYTES) {
+            throw new Error(`custom assertion module graph exceeds ${MAX_ASSERTION_MODULE_BYTES} bytes: ${entryPath}`);
+        }
+        entries.push([
+            `module:${displayDependencyPath(modulePath, baseDir)}`,
+            `sha256:${hashString(source)}`,
+        ]);
+        if (!/\.[cm]?js$/i.test(modulePath))
+            return;
+        let imports;
+        try {
+            [imports] = parse(source.toString('utf8'));
+        }
+        catch {
+            entries.push([
+                `module-parse:${displayDependencyPath(modulePath, baseDir)}`,
+                'unparseable',
+            ]);
+            return;
+        }
+        for (const imported of imports) {
+            if (!imported.n) {
+                if (imported.d >= 0) {
+                    entries.push([
+                        `module-dynamic:${displayDependencyPath(modulePath, baseDir)}:${imported.ss}`,
+                        'non-literal',
+                    ]);
+                }
+                continue;
+            }
+            const resolvedImport = resolveImportedModule(imported.n, modulePath);
+            if (resolvedImport)
+                visit(resolvedImport);
+        }
+    };
+    visit(entryPath);
+    return entries.sort(([left], [right]) => left.localeCompare(right));
+}
+function sampleDependencyEntries(sample, baseDir, includeCustomAssertions) {
+    const entries = [];
+    for (const [index, mock] of (sample.mocks ?? []).entries()) {
+        if (typeof mock.return_file !== 'string' || !mock.return_file)
+            continue;
+        entries.push([
+            `mock:${index}:return_file:${mock.return_file}`,
+            dependencyContentFingerprint(mock.return_file, baseDir),
+        ]);
+    }
+    if (includeCustomAssertions) {
+        for (const [index, path] of customAssertionPaths(sample.assertions).entries()) {
+            for (const [modulePath, fingerprint] of customAssertionModuleEntries(path, baseDir)) {
+                entries.push([
+                    `assertion:${index}:fn:${path}:${modulePath}`,
+                    fingerprint,
+                ]);
+            }
+        }
+    }
+    return entries.sort(([left], [right]) => left.localeCompare(right));
+}
+/**
+ * Content fingerprint for files that can change executor output while the
+ * inline sample contract remains byte-for-byte identical.
+ */
+export function hashSampleExecutionDependencies(sample, baseDir) {
+    const entries = sampleDependencyEntries(sample, baseDir, false);
+    return entries.length > 0 ? hashString(canonicalStringify(entries)).slice(0, 12) : undefined;
+}
+/**
+ * Stable hash of the complete sample measurement contract, including external
+ * mock fixtures and the statically resolvable ESM import graph of custom
+ * assertion modules when a sample base directory is available.
+ */
+export function hashSample(sample, baseDir) {
+    const contract = Object.fromEntries(Object.entries(sample).filter(([key, value]) => key !== 'sample_id' && value !== undefined));
+    const dependencies = sampleDependencyEntries(sample, baseDir, true);
+    return hashString(canonicalStringify({
+        contract,
+        ...(dependencies.length > 0 ? { dependencies } : {}),
+    })).slice(0, 12);
+}

package/dist/eval-core/schema.js

@@ -3,27 +3,13 @@
  * Single source of truth for object structures used across runner, renderer, and server.
  */
 import { computeJudgeAgreement } from '../grading/judge.js';
+import { isToolCallCancelled, isToolCallFailure, isToolCallSuccess, isToolCallUnknown, } from '../shared/tool-call-status.js';
+import { incrementRecordCount } from '../shared/record-count.js';
+import { truncateToolCallsForPersistence, truncateTurnsForPersistence, } from '../shared/trace-projection.js';
 import { safeSliceForJson } from '../util/safe-slice.js';
 function ratioToScore(ratio) {
     return Number((1 + ratio * 4).toFixed(2));
 }
-const MAX_TURN_CONTENT = 2000;
-const MAX_TOOL_OUTPUT = 1000;
-function truncateTurns(turns) {
-    return turns.map((t) => ({
-        ...t,
-        content: safeSliceForJson(t.content, MAX_TURN_CONTENT),
-        ...(t.toolCalls && { toolCalls: truncateToolCalls(t.toolCalls) }),
-    }));
-}
-function truncateToolCalls(toolCalls) {
-    return toolCalls.map((tc) => ({
-        ...tc,
-        output: typeof tc.output === 'string'
-            ? safeSliceForJson(tc.output, MAX_TOOL_OUTPUT)
-            : tc.output,
-    }));
-}
 export function buildVariantResult(execResult, gradeResult, options) {
     const execCostUSD = execResult.costUSD || 0;
     const judgeCostUSD = gradeResult?.judgeCostUSD || 0;
@@ -31,7 +17,12 @@ export function buildVariantResult(execResult, gradeResult, options) {
     const gradeMs = options?.gradeMs || 0;
     const assistantTurns = execResult.turns?.filter((turn) => turn.role === 'assistant').length;
     const toolTurns = execResult.turns?.filter((turn) => turn.role === 'tool').length;
-    const numToolFailures = execResult.toolCalls?.filter((tc) => !tc.success).length;
+    const numToolFailures = execResult.toolCalls?.filter(isToolCallFailure).length;
+    const numToolCancelled = execResult.toolCalls?.filter(isToolCallCancelled).length;
+    const numToolUnknown = execResult.toolCalls?.filter(isToolCallUnknown).length;
+    const comparableToolCalls = Math.max(0, (execResult.toolCalls?.length ?? 0)
+        - (numToolCancelled ?? 0)
+        - (numToolUnknown ?? 0));
     const traceSignals = [
         execResult.turns && execResult.turns.length > 0,
         execResult.toolCalls && execResult.toolCalls.length > 0,
@@ -45,9 +36,15 @@ export function buildVariantResult(execResult, gradeResult, options) {
         durationApiMs: execResult.durationApiMs,
         inputTokens: execResult.inputTokens,
         outputTokens: execResult.outputTokens,
-        totalTokens: execResult.inputTokens + execResult.outputTokens,
+        totalTokens: execResult.inputTokens
+            + execResult.outputTokens
+            + execResult.cacheReadTokens
+            + execResult.cacheCreationTokens,
         cacheReadTokens: execResult.cacheReadTokens,
         cacheCreationTokens: execResult.cacheCreationTokens,
+        ...(execResult.tokenUsageReportedByExecutor === false && {
+            tokenUsageReportedByExecutor: false,
+        }),
         execCostUSD,
         judgeCostUSD,
         // 初值 = exec + judge。diagnostic 在 evaluation-execution.ts 跑完后会把
@@ -61,6 +58,7 @@ export function buildVariantResult(execResult, gradeResult, options) {
         // judge 同理:任一 dim/single/async assertion 走了不报 cost 的 judge executor → grade 整体也不可信
         ...(gradeResult?.judgeCostReportedByExecutor === false && { judgeCostReportedByExecutor: false }),
         numTurns: execResult.numTurns,
+        ...(execResult.attemptCount != null && { attemptCount: execResult.attemptCount }),
         ...(execResult.fullNumTurns != null && { fullNumTurns: execResult.fullNumTurns }),
         ...(execResult.numSubAgents != null && { numSubAgents: execResult.numSubAgents }),
         ...(assistantTurns != null && { assistantTurns }),
@@ -68,10 +66,14 @@ export function buildVariantResult(execResult, gradeResult, options) {
         ...(execResult.toolCalls && execResult.toolCalls.length > 0 && {
             numToolCalls: execResult.toolCalls.length,
             numToolFailures,
-            toolSuccessRate: Number((execResult.toolCalls.filter((tc) => tc.success).length / execResult.toolCalls.length).toFixed(2)),
+            numToolCancelled,
+            numToolUnknown,
+            ...(comparableToolCalls > 0 && {
+                toolSuccessRate: Number((execResult.toolCalls.filter(isToolCallSuccess).length / comparableToolCalls).toFixed(2)),
+            }),
             toolNames: [...new Set(execResult.toolCalls.map((tc) => tc.tool))],
             toolDistribution: execResult.toolCalls.reduce((d, tc) => {
-                d[tc.tool] = (d[tc.tool] || 0) + 1;
+                incrementRecordCount(d, tc.tool);
                 return d;
             }, {}),
         }),
@@ -111,8 +113,12 @@ export function buildVariantResult(execResult, gradeResult, options) {
         ...(options?.factCheck && options.factCheck.totalCount > 0 && { factCheck: options.factCheck }),
         outputPreview: execResult.output ? safeSliceForJson(execResult.output, 200, '') : null,
         ...(execResult.output && { fullOutput: execResult.output }),
-        ...(execResult.turns && execResult.turns.length > 0 && { turns: truncateTurns(execResult.turns) }),
-        ...(execResult.toolCalls && execResult.toolCalls.length > 0 && { toolCalls: truncateToolCalls(execResult.toolCalls) }),
+        ...(execResult.turns && execResult.turns.length > 0 && {
+            turns: truncateTurnsForPersistence(execResult.turns),
+        }),
+        ...(execResult.toolCalls && execResult.toolCalls.length > 0 && {
+            toolCalls: truncateToolCallsForPersistence(execResult.toolCalls),
+        }),
         ...(execResult.mockStats && { mockStats: execResult.mockStats }),
         timing: { execMs, gradeMs, totalMs: execMs + gradeMs },
     };
@@ -122,6 +128,11 @@ export function buildVariantResult(execResult, gradeResult, options) {
  */
 export function buildVariantSummary(entries) {
     const ok = entries.filter((e) => e.ok);
+    const observedTokenEntries = ok.filter((entry) => entry.tokenUsageReportedByExecutor !== false);
+    const tokenUsageCoverageRate = entries.length > 0
+        ? Number((entries.filter((entry) => entry.tokenUsageReportedByExecutor !== false).length
+            / entries.length).toFixed(4))
+        : 1;
     const compositeScores = entries.filter((e) => typeof e.compositeScore === 'number' && e.compositeScore > 0).map((e) => e.compositeScore);
     const assertionScores = entries.filter((e) => e.assertions?.score != null && e.assertions.score > 0).map((e) => e.assertions.score);
     const llmScores = entries.filter((e) => typeof e.llmScore === 'number' && e.llmScore > 0).map((e) => e.llmScore);
@@ -132,9 +143,16 @@ export function buildVariantSummary(entries) {
         errorCount,
         errorRate: entries.length > 0 ? Number((errorCount / entries.length * 100).toFixed(1)) : 0,
         avgDurationMs: ok.length > 0 ? Math.round(ok.reduce((s, e) => s + (e.timing?.totalMs || e.durationMs), 0) / ok.length) : 0,
-        avgInputTokens: ok.length > 0 ? Math.round(ok.reduce((s, e) => s + e.inputTokens, 0) / ok.length) : 0,
-        avgOutputTokens: ok.length > 0 ? Math.round(ok.reduce((s, e) => s + e.outputTokens, 0) / ok.length) : 0,
-        avgTotalTokens: ok.length > 0 ? Math.round(ok.reduce((s, e) => s + e.totalTokens, 0) / ok.length) : 0,
+        avgInputTokens: observedTokenEntries.length > 0
+            ? Math.round(observedTokenEntries.reduce((s, e) => s + e.inputTokens, 0) / observedTokenEntries.length)
+            : 0,
+        avgOutputTokens: observedTokenEntries.length > 0
+            ? Math.round(observedTokenEntries.reduce((s, e) => s + e.outputTokens, 0) / observedTokenEntries.length)
+            : 0,
+        avgTotalTokens: observedTokenEntries.length > 0
+            ? Math.round(observedTokenEntries.reduce((s, e) => s + e.totalTokens, 0) / observedTokenEntries.length)
+            : 0,
+        ...(tokenUsageCoverageRate < 1 && { tokenUsageCoverageRate }),
         totalCostUSD: ok.reduce((s, e) => s + (e.costUSD || 0), 0),
         totalExecCostUSD: ok.reduce((s, e) => s + (e.execCostUSD || 0), 0),
         totalJudgeCostUSD: ok.reduce((s, e) => s + (e.judgeCostUSD || 0), 0),
@@ -180,7 +198,40 @@ export function buildVariantSummary(entries) {
             if (withTools.length === 0)
                 return {};
             const totalToolCalls = withTools.reduce((s, e) => s + (e.numToolCalls || 0), 0);
-            const avgSuccessRate = withTools.reduce((s, e) => s + (e.toolSuccessRate || 0), 0) / withTools.length;
+            const outcomeTotals = withTools.reduce((totals, entry) => {
+                const calls = entry.numToolCalls ?? 0;
+                const explicitUnknown = Math.min(calls, Math.max(0, entry.numToolUnknown ?? 0));
+                const explicitCancelled = Math.min(Math.max(0, calls - explicitUnknown), Math.max(0, entry.numToolCancelled ?? 0));
+                const comparable = Math.max(0, calls - explicitUnknown - explicitCancelled);
+                if (typeof entry.numToolFailures === 'number') {
+                    const failures = Math.min(comparable, Math.max(0, entry.numToolFailures));
+                    totals.comparable += comparable;
+                    totals.failures += failures;
+                    totals.successes += comparable - failures;
+                    totals.cancelled += explicitCancelled;
+                    totals.unknown += explicitUnknown;
+                    return totals;
+                }
+                if (typeof entry.toolSuccessRate === 'number') {
+                    const rate = Math.max(0, Math.min(1, entry.toolSuccessRate));
+                    totals.comparable += comparable;
+                    totals.successes += rate * comparable;
+                    totals.failures += (1 - rate) * comparable;
+                    totals.cancelled += explicitCancelled;
+                    totals.unknown += explicitUnknown;
+                    return totals;
+                }
+                // A call count without failure/success evidence only says execution was
+                // attempted. Treat the whole row as unresolved instead of 0% success.
+                totals.unknown += calls;
+                return totals;
+            }, {
+                comparable: 0,
+                successes: 0,
+                failures: 0,
+                cancelled: 0,
+                unknown: 0,
+            });
             // sum per-sample toolDistribution(真实 call count),
             // 不再用 dedup 后的 toolNames 累加(那只统计"出现该 tool 的 sample 数",不是调用次数)。
             // 旧报告没 toolDistribution 字段时 fallback 到 toolNames 老语义,保兼容性。
@@ -188,19 +239,23 @@ export function buildVariantSummary(entries) {
             for (const e of withTools) {
                 if (e.toolDistribution) {
                     for (const [name, count] of Object.entries(e.toolDistribution)) {
-                        dist[name] = (dist[name] || 0) + count;
+                        incrementRecordCount(dist, name, count);
                     }
                 }
                 else {
                     for (const name of (e.toolNames || [])) {
-                        dist[name] = (dist[name] || 0) + 1;
+                        incrementRecordCount(dist, name);
                     }
                 }
             }
             return {
                 avgToolCalls: Number((totalToolCalls / withTools.length).toFixed(1)),
-                avgToolFailures: Number((withTools.reduce((s, e) => s + (e.numToolFailures || 0), 0) / withTools.length).toFixed(1)),
-                toolSuccessRate: Number(avgSuccessRate.toFixed(2)),
+                avgToolFailures: Number((outcomeTotals.failures / withTools.length).toFixed(1)),
+                avgToolCancelled: Number((outcomeTotals.cancelled / withTools.length).toFixed(1)),
+                avgToolUnknown: Number((outcomeTotals.unknown / withTools.length).toFixed(1)),
+                ...(outcomeTotals.comparable > 0 && {
+                    toolSuccessRate: Number((outcomeTotals.successes / outcomeTotals.comparable).toFixed(2)),
+                }),
                 toolDistribution: dist,
             };
         })(),

package/dist/eval-core/verdict.d.ts

@@ -165,11 +165,15 @@ export declare function medianStabilityCV(report: Report): {
     runs: number;
     cv: number;
 } | null;
+interface VerdictTextOptions {
+    verbose?: boolean;
+    lang?: Lang;
+    /** Managed skill NAME from `omk list`; omitted when the formatter cannot know it safely. */
+    promoteTarget?: string;
+}
 /**
  * Plain-text formatter for the `omk eval` verdict. Stays terse for the
  *  spec — one verdict, rationale bullets, one ship recommendation, and one next step.
  */
-export declare function formatVerdictText(result: VerdictResult, options?: {
-    verbose?: boolean;
-    lang?: Lang;
-}): string;
+export declare function formatVerdictText(result: VerdictResult, options?: VerdictTextOptions): string;
+export {};

package/dist/eval-core/verdict.js

@@ -31,6 +31,7 @@ import { evaluateLayerGates } from './layer-gates.js';
 import { ciLevelLabel } from './bootstrap.js';
 import { analyzeJudgeIndependence } from './judge-independence.js';
 import { MIN_HOLDOUT_SUBSET } from './holdout.js';
+import { shellQuoteArg } from '../shared/shell-quote.js';
 /**
  * Below this sample count a non-significant diff is read as UNDERPOWERED
  * (only large effects are detectable) rather than NOISE. Matches the
@@ -584,13 +585,26 @@ function recommendation(level, _perPair, lang = 'en') {
             return 'ADD A CONTROL — single-variant report. Re-run with --control baseline --treatment <name>.';
     }
 }
-function releaseNextStep(level, lang) {
+function nextStepTreatmentName(result) {
+    if (result.representative?.treatment)
+        return result.representative.treatment;
+    if (result.level === 'SOLO')
+        return result.variants[0] ?? '<name>';
+    return result.variants[1] ?? '<name>';
+}
+function promoteCommand(target) {
+    return target ? `omk promote ${shellQuoteArg(target)}` : 'omk promote <name>';
+}
+function releaseNextStep(result, lang, promoteTarget) {
+    const treatment = nextStepTreatmentName(result);
+    const treatmentArg = shellQuoteArg(treatment);
+    const promote = promoteCommand(promoteTarget);
     if (lang === 'zh') {
-        switch (level) {
+        switch (result.level) {
             case 'PROGRESS':
-                return '可以进入发布流程：请留存本次报告作为发布证据；如果这是受管 skill，再运行 `omk promote`。';
+                return `可以发布：按你的正常发布流程发布，并留存本次报告作为发布证据；如果这是受管 skill，继续运行 \`${promote}\` 记录接受决定（name 以 \`omk list\` 的 NAME 为准）。`;
             case 'CAUTIOUS':
-                return '先看触发的告警（分层门控、评委分歧、稳定性或 holdout），修完再重跑。';
+                return '不要直接发布；先看触发的告警（分层门控、评委分歧、稳定性或 holdout），修完再重跑。';
             case 'REGRESS':
                 return '不要发布；定位最差层和失败用例，修复后重跑。';
             case 'NOISE':
@@ -598,14 +612,14 @@ function releaseNextStep(level, lang) {
             case 'UNDERPOWERED':
                 return '把样本数加到至少 20，或先按当前规模 2× 扩充后重跑。';
             case 'SOLO':
-                return '补一个 baseline 对照，再跑 `omk eval --control baseline --treatment <名字>`。';
+                return `补一个 baseline 对照，再跑 \`omk eval --control baseline --treatment ${treatmentArg}\`。`;
         }
     }
-    switch (level) {
+    switch (result.level) {
         case 'PROGRESS':
-            return 'ready for release: keep this report as release evidence; for a managed skill, run `omk promote`.';
+            return `ship through your normal release path and keep this report as release evidence; for a managed skill, run \`${promote}\` to record acceptance (name is the NAME from \`omk list\`).`;
         case 'CAUTIOUS':
-            return 'inspect the warnings (layer gates, judge dissent, stability, or holdout), fix them, then re-run.';
+            return 'do not ship directly; inspect the warnings (layer gates, judge dissent, stability, or holdout), fix them, then re-run.';
         case 'REGRESS':
             return 'do not ship; inspect the weakest layer and failing samples, fix them, then re-run.';
         case 'NOISE':
@@ -613,7 +627,7 @@ function releaseNextStep(level, lang) {
         case 'UNDERPOWERED':
             return 'increase the sample set to at least 20, or roughly 2x the current size, then re-run.';
         case 'SOLO':
-            return 'add a baseline control and re-run `omk eval --control baseline --treatment <name>`.';
+            return `add a baseline control and re-run \`omk eval --control baseline --treatment ${treatmentArg}\`.`;
     }
 }
 /**
@@ -642,7 +656,7 @@ export function formatVerdictText(result, options = {}) {
         lines.push(zh ? `  知识缺口：${result.rationale.gapSignal}` : `  Gap signal:    ${result.rationale.gapSignal}`);
     if (result.rationale.shipRecommendation) {
         lines.push(`  ${zh ? recommendation(result.level, [], 'zh') : result.rationale.shipRecommendation}`);
-        lines.push(zh ? `  下一步：${releaseNextStep(result.level, 'zh')}` : `  Next: ${releaseNextStep(result.level, 'en')}`);
+        lines.push(zh ? `  下一步：${releaseNextStep(result, 'zh', options.promoteTarget)}` : `  Next: ${releaseNextStep(result, 'en', options.promoteTarget)}`);
     }
     if (options.verbose && result.perPair && result.perPair.length > 1) {
         lines.push('');

package/dist/eval-workflows/batch-evaluation-workflow.d.ts

@@ -58,7 +58,7 @@ export declare function buildBatchVariantSpecs(entry: {
     name: string;
     skillPath: string;
 }, variantAllowedSkills?: Record<string, string[]>): VariantSpec[];
-export declare function buildBatchEvaluationReport({ batchRunId, skillDir, skillEntries, skillResults, model, judgeModel, noJudge, executorName, judgeExecutorName, project, owner, tags, concurrency, timeoutMs, totalCostUSD, repeat, judgeModels, noCache, bootstrap, bootstrapSamples, lengthDebias, budget, }: {
+export declare function buildBatchEvaluationReport({ batchRunId, skillDir, skillEntries, skillResults, model, judgeModel, noJudge, executorName, judgeExecutorName, project, owner, tags, concurrency, timeoutMs, totalCostUSD, repeat, judgeModels, noCache, bootstrap, bootstrapSamples, lengthDebias, budget, strictBaseline, }: {
     batchRunId: string;
     skillDir: string;
     skillEntries: Array<{
@@ -85,6 +85,7 @@ export declare function buildBatchEvaluationReport({ batchRunId, skillDir, skill
     bootstrapSamples?: number;
     lengthDebias?: boolean;
     budget?: import('../types/index.js').EvalBudget;
+    strictBaseline?: boolean;
 }): {
     report: BatchEvaluationReport;
     job: import('../types/index.js').EvaluationJob;