oh-my-knowledge 0.51.1 → 0.52.0

package/dist/authoring/sample-fixer.js

@@ -20,8 +20,17 @@ const FIX_SYSTEM_PROMPT = `你是一个评测用例修复专家。根据诊断
 输出一个 **JSON 数组**,包含所有待修复 sample（修改过的和原样保留的都要包含）。
 第一字符 \`[\`，最后 \`]\`，不要用 \`\`\`json\`\`\` 围栏，不要寒暄。
 如果判断某条是 LLM 行为问题不需要改,也原样放进数组。`;
-function sanitizeFixedSamples(samples, skillContent) {
-    sanitizeGeneratedSamples(samples, { skillContent });
+const MOCKLESS_FIX_OVERRIDE = `
+
+目标执行器不支持工具调用拦截。不得新增或保留 mocks、mocksStrict、
+mock_hit、tools_called、tools_count_min、tool_input_contains、tool_output_contains。
+environment 仅表示题设上下文，不得当作已物化 fixture；负向安全断言可以保留。`;
+function sanitizeFixedSamples(samples, skillContent, mockless) {
+    sanitizeGeneratedSamples(samples, {
+        skillContent,
+        mockless,
+        preserveMocklessEnvironment: true,
+    });
     return samples;
 }
 const FIXABLE_SAMPLE_FIELDS = new Set([
@@ -97,7 +106,7 @@ function parseFixedSamples(text) {
     return null;
 }
 export async function fixSamples(options) {
-    const { skillContent, samples, report, treatmentKey, executor, model, maxAttemptsPerSample = 2 } = options;
+    const { skillContent, samples, report, treatmentKey, executor, model, maxAttemptsPerSample = 2, mockless = false, } = options;
     const sampleMap = new Map(samples.map((s) => [s.sample_id, s]));
     // Collect all fixable samples into one batch
     const fixContexts = [];
@@ -185,7 +194,12 @@ ${sampleSections}
     let incurredCostUSD = 0;
     let incurredCostReported = false;
     try {
-        const result = await executor({ model, system: FIX_SYSTEM_PROMPT, prompt, timeoutMs: 300_000 });
+        const result = await executor({
+            model,
+            system: mockless ? `${FIX_SYSTEM_PROMPT}${MOCKLESS_FIX_OVERRIDE}` : FIX_SYSTEM_PROMPT,
+            prompt,
+            timeoutMs: 300_000,
+        });
         incurredCostUSD = result.costUSD;
         incurredCostReported = result.costReported !== false;
         if (!result.ok) {
@@ -226,7 +240,7 @@ ${sampleSections}
                 })),
             };
         }
-        const sanitizedFixedArr = sanitizeFixedSamples(fixedArr, skillContent);
+        const sanitizedFixedArr = sanitizeFixedSamples(fixedArr, skillContent, mockless);
         const outOfScope = sanitizedFixedArr.find((fixed) => {
             const sid = fixed.sample_id;
             const original = sampleMap.get(sid);

package/dist/cli/commands/sample.js

@@ -274,7 +274,7 @@ async function runSampleFix(args, flags, lang) {
         throw new CliExit(1);
     }
     process.stderr.write(lang === 'zh' ? `🔧 发现 ${sampleDesignCount} 条 sample_design 失败，开始修复...\n` : `🔧 Found ${sampleDesignCount} sample_design failure(s), fixing...\n`);
-    const { createExecutor } = await import('../../executors/index.js');
+    const { createExecutor, executorSupportsSampleMocks, } = await import('../../executors/index.js');
     const exec = createExecutor(executorName);
     const executorFn = async (opts) => {
         const result = await exec({
@@ -298,6 +298,7 @@ async function runSampleFix(args, flags, lang) {
         treatmentKey: treatmentName,
         executor: executorFn,
         model,
+        mockless: !executorSupportsSampleMocks(executorName),
     });
     let writtenFiles = [];
     if (result.fixedCount > 0) {
@@ -363,7 +364,13 @@ export async function runSampleFromTraces(flags, lang) {
         ? `🔭 发现 ${items.length} 个${flags.skill ? ` ${flags.skill} 的` : ''}失败信号，正在生成评测用例草稿...\n`
         : `🔭 Found ${items.length}${flags.skill ? ` ${flags.skill}` : ''} failure signal(s); generating regression-sample drafts...\n`);
     try {
-        const { samples, costUSD } = await generateSamplesFromTraces({ items, count, model, executorName });
+        const { samples, costUSD } = await generateSamplesFromTraces({
+            items,
+            count,
+            model,
+            executorName,
+            noMock: flags['no-mock'],
+        });
         const cost = costUSD > 0 ? ` $${costUSD.toFixed(4)}` : '';
         if (samples.length === 0) {
             // The model conservatively skipped every signal (noise / unreproducible). That's a
@@ -663,8 +670,8 @@ export default class Sample extends BaseCommand {
         }),
         'no-mock': Flags.boolean({
             description: bilingual({
-                zh: '不生成 mocks，eval 时所有工具调用真实执行。',
-                en: 'Skip mock generation; all tool calls execute for real during eval.',
+                zh: '不生成 mocks。执行器不支持工具拦截时会自动启用，避免产生必然失败的 mock_hit。',
+                en: 'Skip mocks. Automatically enabled when the executor cannot intercept tools, preventing impossible mock_hit assertions.',
             }),
             default: false,
         }),

package/dist/eval-core/evaluation-execution.js

@@ -258,8 +258,17 @@ export async function executeTasks({ tasks, executor, executorName, model, noJud
             }
         }
         let factCheck;
-        if (execResult.ok && execResult.output && task.cwd) {
-            factCheck = checkFacts(execResult.output, resolve(task.cwd));
+        if (execResult.ok && execResult.output) {
+            const sharedEvidence = {
+                ...(task._sample.context && { context: task._sample.context }),
+                ...(task._sample.environment?.files_available?.length && {
+                    declaredFiles: task._sample.environment.files_available,
+                }),
+                // Resolve exactly like the executor's cwd. samplesBaseDir is for bundle
+                // assets such as mocks, not for changing Sample.cwd path semantics.
+                ...(task._sample.cwd && { cwd: resolve(task._sample.cwd) }),
+            };
+            factCheck = checkFacts(execResult.output, sharedEvidence);
         }
         const sampleResults = ownRecordValue(results, task.sample_id)
             ?? setOwnRecordValue(results, task.sample_id, {});

package/dist/eval-core/fact-checker.d.ts

@@ -14,11 +14,24 @@ export interface FactCheckResult {
     totalCount: number;
     verifiedRate: number;
 }
+export interface FactCheckEvidence {
+    /** Shared sample fixture root. Arm-specific execution directories must not be used. */
+    cwd?: string;
+    /** Facts supplied to both arms in the sample context. */
+    context?: string;
+    /** Declarative fixture paths from sample.environment.files_available. */
+    declaredFiles?: string[];
+}
 /**
  * Extract file path claims from agent output text.
  */
 export declare function extractPathClaims(output: string): string[];
 /**
- * Check facts in agent output by verifying file paths exist in cwd.
+ * Check file-path facts against sample-level evidence shared by every arm.
+ *
+ * A string keeps the legacy direct-filesystem API for callers outside the
+ * evaluation pipeline. The pipeline passes structured evidence and deliberately
+ * excludes each artifact's execution cwd, because that directory is not a
+ * comparable source of truth across control and treatment arms.
  */
-export declare function checkFacts(output: string, cwd: string): FactCheckResult;
+export declare function checkFacts(output: string, evidence: string | FactCheckEvidence): FactCheckResult;

package/dist/eval-core/fact-checker.js

@@ -22,6 +22,10 @@ const IGNORE_PATTERNS = [
     /^\.git\//,
     /^index\.\w+$/,
 ];
+// Product/runtime names that happen to end in a supported source extension.
+const NON_PATH_REFERENCES = new Set([
+    'node.js',
+]);
 /**
  * Extract file path claims from agent output text.
  */
@@ -36,35 +40,86 @@ export function extractPathClaims(output) {
             path = path.replace(/[.,;:!?）)]+$/, '');
             // Clean trailing backtick/quote
             path = path.replace(/[`'"]+$/, '');
-            if (path.length > 3 && !IGNORE_PATTERNS.some((p) => p.test(path))) {
+            if (path.length > 3
+                && !NON_PATH_REFERENCES.has(path.toLowerCase())
+                && !IGNORE_PATTERNS.some((p) => p.test(path))) {
                 paths.add(path);
             }
         }
     }
     return [...paths];
 }
+function normalizeEvidencePath(path) {
+    return path
+        .trim()
+        .replace(/^['"`]+|['"`]+$/g, '')
+        .replace(/\\/g, '/')
+        .replace(/^(?:\$SKILL_DIR|~)\//, '')
+        .replace(/^\.\//, '')
+        .replace(/\/+$/, '');
+}
+function isRootedEvidencePath(path) {
+    return path.startsWith('/') || /^[a-zA-Z]:\//.test(path);
+}
+function refersToSamePath(left, right) {
+    const a = normalizeEvidencePath(left);
+    const b = normalizeEvidencePath(right);
+    return a === b
+        || (isRootedEvidencePath(a) && !isRootedEvidencePath(b) && a.endsWith(`/${b}`))
+        || (isRootedEvidencePath(b) && !isRootedEvidencePath(a) && b.endsWith(`/${a}`));
+}
 /**
- * Check facts in agent output by verifying file paths exist in cwd.
+ * Check file-path facts against sample-level evidence shared by every arm.
+ *
+ * A string keeps the legacy direct-filesystem API for callers outside the
+ * evaluation pipeline. The pipeline passes structured evidence and deliberately
+ * excludes each artifact's execution cwd, because that directory is not a
+ * comparable source of truth across control and treatment arms.
  */
-export function checkFacts(output, cwd) {
+export function checkFacts(output, evidence) {
     const pathClaims = extractPathClaims(output);
-    const root = resolve(cwd);
-    const claims = pathClaims.map((path) => {
-        const fullPath = resolve(root, path);
-        const relativePath = relative(root, fullPath);
-        const insideCwd = relativePath === ''
-            || (!relativePath.startsWith('..') && !isAbsolute(relativePath));
-        const exists = insideCwd && existsSync(fullPath);
-        return {
-            type: 'file-path',
-            value: path,
-            verified: exists,
-            ...(!exists && {
-                evidence: insideCwd
-                    ? `${fullPath} not found`
-                    : `${path} is outside the evaluation cwd`,
-            }),
-        };
+    const sources = typeof evidence === 'string'
+        ? { cwd: evidence }
+        : evidence;
+    const root = sources.cwd ? resolve(sources.cwd) : null;
+    const contextClaims = sources.context
+        ? extractPathClaims(sources.context)
+        : [];
+    const declaredFiles = sources.declaredFiles ?? [];
+    const claims = pathClaims.flatMap((path) => {
+        if (declaredFiles.some((declared) => refersToSamePath(path, declared))) {
+            return [{
+                    type: 'file-path',
+                    value: path,
+                    verified: true,
+                    evidence: 'source=context(sample.environment.files_available)',
+                }];
+        }
+        if (root) {
+            const fullPath = resolve(root, path);
+            const relativePath = relative(root, fullPath);
+            const insideCwd = relativePath === ''
+                || (!relativePath.startsWith('..') && !isAbsolute(relativePath));
+            const exists = insideCwd && existsSync(fullPath);
+            return [{
+                    type: 'file-path',
+                    value: path,
+                    verified: exists,
+                    evidence: insideCwd
+                        ? `source=runtime-filesystem; ${fullPath} ${exists ? 'exists' : 'not found'}`
+                        : `source=runtime-filesystem; ${path} is outside the evaluation cwd`,
+                }];
+        }
+        if (contextClaims.some((contextPath) => refersToSamePath(path, contextPath))) {
+            return [{
+                    type: 'file-path',
+                    value: path,
+                    verified: true,
+                    evidence: 'source=context(sample.context)',
+                }];
+        }
+        // Without a shared fixture, absence from context is unknown rather than false.
+        return [];
     });
     const verifiedCount = claims.filter((c) => c.verified).length;
     const totalCount = claims.length;

package/dist/eval-core/mock-hook.cjs

@@ -92,8 +92,47 @@ function anyStringContains(obj, needle) {
   return false;
 }
 
+const BUILTIN_TOOL_ALIASES = {
+  bash: 'Bash',
+  shell: 'Bash',
+  exec_command: 'Bash',
+  command_execution: 'Bash',
+  read: 'Read',
+  file_read: 'Read',
+  grep: 'Grep',
+  edit: 'Edit',
+  apply_patch: 'Edit',
+  file_change: 'Edit',
+  write: 'Write',
+  file_write: 'Write',
+  view_image: 'ViewImage',
+  viewimage: 'ViewImage',
+  write_stdin: 'WriteStdin',
+  writestdin: 'WriteStdin',
+  web_search: 'WebSearch',
+  websearch: 'WebSearch',
+};
+
+function canonicalToolName(name) {
+  const sourceName = String(name);
+  const builtin = BUILTIN_TOOL_ALIASES[sourceName.toLowerCase()];
+  if (builtin) return builtin;
+  const parts = sourceName.split('__').filter(Boolean);
+  if (parts[0] === 'mcp' && parts.length > 2) {
+    const providerParts = parts.slice(1, -1);
+    if (providerParts[0] === 'codex_apps' && providerParts.length > 1) providerParts.shift();
+    return providerParts.join('.') + '.' + parts[parts.length - 1];
+  }
+  return sourceName;
+}
+
+function toolIdentityMatches(expectedName, runtimeName) {
+  return expectedName === runtimeName
+    || canonicalToolName(expectedName) === canonicalToolName(runtimeName);
+}
+
 function isMockHit(mock, toolName, toolInput) {
-  if (mock.tool !== '*' && mock.tool !== toolName) return false;
+  if (mock.tool !== '*' && !toolIdentityMatches(mock.tool, toolName)) return false;
   const m = mock.match;
   if (!m) return true;
   const ti = toolInput || {};

package/dist/eval-core/mocks-runtime.js

@@ -18,6 +18,7 @@ import { homedir, tmpdir } from 'node:os';
 import { dirname, isAbsolute, join, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { incrementRecordCount, setOwnRecordValue } from '../shared/record-count.js';
+import { toolIdentityMatches } from '../shared/tool-identity.js';
 // ─── Match logic ────────────────────────────────────────────────────────────
 function expandHome(p) {
     if (p.startsWith('~/'))
@@ -112,7 +113,7 @@ function anyStringContains(obj, needle) {
 }
 /** 单条 mock 是否命中给定 tool 调用。 */
 export function isMockHit(mock, toolName, toolInput) {
-    if (mock.tool !== '*' && mock.tool !== toolName)
+    if (mock.tool !== '*' && !toolIdentityMatches(mock.tool, toolName))
         return false;
     const m = mock.match;
     if (!m)
@@ -414,8 +415,44 @@ function anyStringContains(obj, needle) {
   if (typeof obj === 'object' && obj !== null) return Object.values(obj).some((v) => anyStringContains(v, needle));
   return false;
 }
+const BUILTIN_TOOL_ALIASES = {
+  bash: 'Bash',
+  shell: 'Bash',
+  exec_command: 'Bash',
+  command_execution: 'Bash',
+  read: 'Read',
+  file_read: 'Read',
+  grep: 'Grep',
+  edit: 'Edit',
+  apply_patch: 'Edit',
+  file_change: 'Edit',
+  write: 'Write',
+  file_write: 'Write',
+  view_image: 'ViewImage',
+  viewimage: 'ViewImage',
+  write_stdin: 'WriteStdin',
+  writestdin: 'WriteStdin',
+  web_search: 'WebSearch',
+  websearch: 'WebSearch',
+};
+function canonicalToolName(name) {
+  const sourceName = String(name);
+  const builtin = BUILTIN_TOOL_ALIASES[sourceName.toLowerCase()];
+  if (builtin) return builtin;
+  const parts = sourceName.split('__').filter(Boolean);
+  if (parts[0] === 'mcp' && parts.length > 2) {
+    const providerParts = parts.slice(1, -1);
+    if (providerParts[0] === 'codex_apps' && providerParts.length > 1) providerParts.shift();
+    return providerParts.join('.') + '.' + parts[parts.length - 1];
+  }
+  return sourceName;
+}
+function toolIdentityMatches(expectedName, runtimeName) {
+  return expectedName === runtimeName
+    || canonicalToolName(expectedName) === canonicalToolName(runtimeName);
+}
 function isMockHit(mock, toolName, toolInput) {
-  if (mock.tool !== '*' && mock.tool !== toolName) return false;
+  if (mock.tool !== '*' && !toolIdentityMatches(mock.tool, toolName)) return false;
   const m = mock.match;
   if (!m) return true;
   const ti = toolInput || {};

package/dist/eval-core/task-planner.d.ts

@@ -1,8 +1,8 @@
 import type { Artifact, Sample, SampleEnvironment, Task } from '../types/index.js';
 /**
  * 把 sample.environment 渲染成自然语言段落,放在用户 prompt 前。
- * 让 LLM 读到"环境已就绪",跳过 Glob / find / which / Read 这些环境探测,
- * 直接进入 skill 描述的工作流 — 评测信号纯,mock 设计也简化(不用 mock 探测命令)。
+ * 这是题设上下文,不会修改 PATH、物化文件或改变 runtime;LLM 可据此跳过
+ * which / test -f 等可用性探测,直接进入 skill 描述的工作流。
  *
  * 输出 null 表示 sample 没声明 environment,prompt 不变。
  */

package/dist/eval-core/task-planner.js

@@ -1,7 +1,7 @@
 /**
  * 把 sample.environment 渲染成自然语言段落,放在用户 prompt 前。
- * 让 LLM 读到"环境已就绪",跳过 Glob / find / which / Read 这些环境探测,
- * 直接进入 skill 描述的工作流 — 评测信号纯,mock 设计也简化(不用 mock 探测命令)。
+ * 这是题设上下文,不会修改 PATH、物化文件或改变 runtime;LLM 可据此跳过
+ * which / test -f 等可用性探测,直接进入 skill 描述的工作流。
  *
  * 输出 null 表示 sample 没声明 environment,prompt 不变。
  */
@@ -10,26 +10,26 @@ export function renderEnvironmentSection(env) {
         return null;
     const lines = [];
     if (env.cli_available && env.cli_available.length > 0) {
-        lines.push('- 已安装 CLI(已在 PATH,无需 `which` / `command -v` / `type` 探测):');
+        lines.push('- 题设声明可用的 CLI（仅作上下文，不修改 PATH）：');
         for (const c of env.cli_available)
             lines.push(`  - \`${c}\``);
     }
     if (env.files_available && env.files_available.length > 0) {
-        lines.push('- 已存在文件(无需 Glob / `Read` / `test -f` 探测):');
+        lines.push('- 题设引用的文件路径（仅作上下文，不会在 cwd 物化）：');
         for (const f of env.files_available)
             lines.push(`  - \`${f}\``);
     }
     if (env.notes && env.notes.trim()) {
-        lines.push(`- 备注:${env.notes.trim()}`);
+        lines.push(`- 备注：${env.notes.trim()}`);
     }
     if (lines.length === 0)
         return null;
     return [
-        '## 评测环境前置(已就绪,无需探测)',
+        '## 题设环境声明（仅作上下文）',
         '',
         ...lines,
         '',
-        '请直接进入 skill 描述的主流程,**不要做环境检查 / `Glob` / `find` / `which` / `test -f` 等探测**。',
+        '请按以上题设进入 skill 描述的主流程，**不要额外做 `find` / `which` / `test -f` 等可用性探测**。这些声明不会自动创建文件或修改 runtime 环境。',
     ].join('\n');
 }
 export function buildTasks(samples, variants, skills) {

package/dist/eval-workflows/evaluation-pipeline.js

@@ -28,6 +28,7 @@ import { finalizeSuccessfulRun, initializeEvaluationRunState, persistFailedJob,
 import { finalizeEvaluationReport } from './evaluation-pipeline/report-finalize.js';
 import { emitIsolationWarnings, emitPowerWarnings } from './evaluation-pipeline/preflight-warnings.js';
 import { ownRecordValue, setOwnRecordValue } from '../shared/record-count.js';
+import { assertSamplesCompatibleWithExecutor } from '../executors/capabilities.js';
 // 兼容 re-export:测试与 run-evaluation.ts 动态 import 仍打 evaluation-pipeline.js
 export { buildPowerWarnings, buildIsolationWarnings } from './evaluation-pipeline/preflight-warnings.js';
 export { _computeTestSetHashForTest } from './evaluation-pipeline/test-set-hash.js';
@@ -35,6 +36,7 @@ export async function executeEvaluationPipeline({ samplesPath, samplesBaseDir, s
 // requires 现在由 runEvaluation 上游传给 doctor 处理; eval-pipeline 不再用
 // 但保留接口字段,避免破 programmatic API (类型层面接收, 内部忽略)
 requires: _requires, layeredStats = false, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias = true, budget, strictBaseline, runId, lang = 'zh', effort, noDiagnostic, }) {
+    assertSamplesCompatibleWithExecutor(samples, executorName, lang);
     const variantNames = artifacts.map((artifact) => artifact.name);
     const runState = await initializeEvaluationRunState({
         samplesPath,

package/dist/eval-workflows/run-evaluation.js

@@ -1,6 +1,6 @@
 import { resolve } from 'node:path';
 import { DEFAULT_OUTPUT_DIR, persistReport } from '../eval-core/evaluation-reporting.js';
-import { createExecutor } from '../executors/index.js';
+import { assertSamplesCompatibleWithExecutor, createExecutor, } from '../executors/index.js';
 import { discoverBatchSkills } from '../inputs/skill-loader.js';
 import { confidenceInterval, tTest, effectSize } from '../eval-core/statistics.js';
 import { executeBatchEvaluationRuns, buildBatchVariantSpecs } from './batch-evaluation-workflow.js';
@@ -26,6 +26,7 @@ export async function runEvaluation({ samplesPath, skillDir, variantSpecs = [],
         mcpConfig,
         strictBaseline,
     });
+    assertSamplesCompatibleWithExecutor(samples, executorName, lang);
     // doctor 强制门禁: skill 静态结构 + 元数据 + 依赖 + 用例契约。
     // 在 dryRun 分支之前跑, 让 dry-run 也得到 doctor 覆盖(保护 garbage-in 的 verdict)。
     // 默认强制启用; --skip-doctor 提供 escape hatch,典型场景是评测环境用 mock/stub

package/dist/executors/capabilities.d.ts

@@ -0,0 +1,15 @@
+import type { ExecutorFn, ExecutorInput, Sample } from '../types/index.js';
+export type SampleMockSupport = 'native-hooks' | 'delegated-script' | 'unsupported';
+export interface ExecutorCapabilities {
+    sampleMocks: SampleMockSupport;
+}
+/**
+ * Custom script executors receive the OMK_MOCK_* protocol environment and own
+ * the final adapter. Built-ins are explicit so unsupported runtimes can never
+ * silently turn mock assertions into model failures.
+ */
+export declare function getExecutorCapabilities(executorName: string): ExecutorCapabilities;
+export declare function executorSupportsSampleMocks(executorName: string): boolean;
+export declare function assertSamplesCompatibleWithExecutor(samples: Sample[], executorName: string, lang?: 'zh' | 'en'): void;
+export declare function assertExecutorInputCapabilities(executorName: string, input: ExecutorInput): void;
+export declare function enforceExecutorCapabilities(executorName: string, executor: ExecutorFn): ExecutorFn;

package/dist/executors/capabilities.js

@@ -0,0 +1,64 @@
+const BUILTIN_CAPABILITIES = {
+    claude: { sampleMocks: 'native-hooks' },
+    'claude-sdk': { sampleMocks: 'native-hooks' },
+    codex: { sampleMocks: 'unsupported' },
+    'codex-sdk': { sampleMocks: 'unsupported' },
+    gemini: { sampleMocks: 'unsupported' },
+    'anthropic-api': { sampleMocks: 'unsupported' },
+    'openai-api': { sampleMocks: 'unsupported' },
+};
+/**
+ * Custom script executors receive the OMK_MOCK_* protocol environment and own
+ * the final adapter. Built-ins are explicit so unsupported runtimes can never
+ * silently turn mock assertions into model failures.
+ */
+export function getExecutorCapabilities(executorName) {
+    return BUILTIN_CAPABILITIES[executorName]
+        ?? { sampleMocks: 'delegated-script' };
+}
+export function executorSupportsSampleMocks(executorName) {
+    return getExecutorCapabilities(executorName).sampleMocks !== 'unsupported';
+}
+function unsupportedMocksMessage(executorName, sampleIds, lang) {
+    const ids = sampleIds.slice(0, 8).join(', ');
+    const overflow = sampleIds.length > 8
+        ? lang === 'zh'
+            ? ` 等 ${sampleIds.length} 条`
+            : ` and ${sampleIds.length - 8} more`
+        : '';
+    if (lang === 'en') {
+        return `Executor "${executorName}" does not support Sample.mocks tool interception. `
+            + `Continuing would make mock_hit assertions structurally impossible and create false evidence. `
+            + `Affected samples: ${ids}${overflow}. `
+            + 'Regenerate them with "omk sample --no-mock", remove mocks/mock_hit, '
+            + 'or evaluate with claude/claude-sdk.';
+    }
+    return `执行器「${executorName}」不支持 Sample.mocks 工具拦截。`
+        + `继续运行会让 mock_hit 在结构上必然失败并产生伪证据。`
+        + `受影响用例：${ids}${overflow}。`
+        + '请用「omk sample --no-mock」重新生成、删除 mocks/mock_hit，'
+        + '或改用 claude/claude-sdk 评测。';
+}
+export function assertSamplesCompatibleWithExecutor(samples, executorName, lang = 'zh') {
+    if (executorSupportsSampleMocks(executorName))
+        return;
+    const affected = samples
+        .filter((sample) => Array.isArray(sample.mocks) && sample.mocks.length > 0)
+        .map((sample) => sample.sample_id);
+    if (affected.length === 0)
+        return;
+    throw new Error(unsupportedMocksMessage(executorName, affected, lang));
+}
+export function assertExecutorInputCapabilities(executorName, input) {
+    if (executorSupportsSampleMocks(executorName)
+        || !Array.isArray(input.mocks)
+        || input.mocks.length === 0)
+        return;
+    throw new Error(unsupportedMocksMessage(executorName, ['<programmatic-input>'], 'zh'));
+}
+export function enforceExecutorCapabilities(executorName, executor) {
+    return async (input) => {
+        assertExecutorInputCapabilities(executorName, input);
+        return executor(input);
+    };
+}

package/dist/executors/index.d.ts

@@ -2,4 +2,5 @@ import type { ExecutorFn } from '../types/index.js';
 import { extractAgentTrace } from './claude-sdk-trace.js';
 import { createScriptExecutor } from './script.js';
 export { extractAgentTrace, createScriptExecutor };
+export { assertExecutorInputCapabilities, assertSamplesCompatibleWithExecutor, executorSupportsSampleMocks, getExecutorCapabilities, } from './capabilities.js';
 export declare function createExecutor(name: string): ExecutorFn;

package/dist/executors/index.js

@@ -7,6 +7,7 @@ import { codexSdkExecutor } from './codex-sdk.js';
 import { geminiExecutor } from './gemini.js';
 import { openAiApiExecutor } from './openai-api.js';
 import { createScriptExecutor } from './script.js';
+import { enforceExecutorCapabilities } from './capabilities.js';
 // 命名一致性:provider HTTP 路径统一用 `<vendor>-api`(`anthropic-api` / `openai-api`),
 // vendor coding agent CLI 用 vendor 名(`claude` / `codex`)。`openai` 这个不带 -api 后缀的
 // 旧 alias 历史上指 openai-cli 子进程实现,删除后不再设别名 — 用 `--executor openai-api`。
@@ -20,9 +21,11 @@ const EXECUTOR_REGISTRY = {
     'openai-api': openAiApiExecutor,
 };
 export { extractAgentTrace, createScriptExecutor };
+export { assertExecutorInputCapabilities, assertSamplesCompatibleWithExecutor, executorSupportsSampleMocks, getExecutorCapabilities, } from './capabilities.js';
 export function createExecutor(name) {
     if (name.trim().length === 0) {
         throw new Error('executor name or script command is required');
     }
-    return EXECUTOR_REGISTRY[name] || createScriptExecutor(name);
+    const executor = EXECUTOR_REGISTRY[name] || createScriptExecutor(name);
+    return enforceExecutorCapabilities(name, executor);
 }