oh-my-knowledge 0.51.1 → 0.51.2

package/dist/assets/agent-skills/omk/SKILL.md

@@ -123,6 +123,8 @@ omk sample skills/my-skill/SKILL.md --focus "重点覆盖搜索失败 / 权限
 omk sample --batch
 ```
 
+目标执行器不支持工具拦截时，`omk sample` 会自动生成无 mock 用例。当前 `codex` / `codex-sdk` 属于这种情况；不要手工补 `mocks` 或 `mock_hit`。已有 mocks 用例会被 `omk eval` 在模型调用前拒绝，避免把执行器能力缺口误判成模型失败。`environment.files_available` 只提供题设上下文，不会在 `cwd` 物化文件。
+
 输出位置：目录 skill（`<skill>/SKILL.md`）→ `<skill>/.omk/samples.json`（标准）；扁平 `.md` 单次生成 → 当前目录 `eval-samples.json`（项目级兜底）；扁平 `.md` 的 `--batch` 兼容生成 `<skill-dir>/<name>.eval-samples.json`。
 
 ### 观测真实使用

package/dist/assets/agent-skills/omk/references/commands.md

@@ -602,7 +602,7 @@ omk sample [skillPath] [flags]
 - `--from-traces` `boolean`:from-traces 模式：从 observe inbox 的失败信号回流生成评测用例草稿（provenance: production-trace），落草稿待人工 review。
 - `--lang` `option` (默认 `zh`):输出语言 zh|en，优先级 CLI > OMK_LANG env > zh。
 - `--model` `option`:生成 LLM model 名。Codex 自动读取本机配置；也可用 OMK_MODEL 设置环境偏好。
-- `--no-mock` `boolean`:不生成 mocks，eval 时所有工具调用真实执行。
+- `--no-mock` `boolean`:不生成 mocks。执行器不支持工具拦截时会自动启用，避免产生必然失败的 mock_hit。
 - `--observations-dir` `option`:observe inbox 目录（from-traces 模式用），默认项目 .omk/observe-inbox。
 - `--reports-dir` `option`:报告目录（fix 模式用），默认 ~/.oh-my-knowledge/reports。
 - `--skill` `option`:仅从指定 skill 的 observe inbox 信号生成草稿（仅 from-traces 模式用）。
@@ -689,7 +689,7 @@ omk studio --port 8080 --no-open
 | `construct` | 否 | 测的是什么构念 |
 | `provenance` | 否 | 用例来源（`omk sample` 自动打） |
 | `mocks` | 否 | 工具调用 mock 返回（sandbox 评测） |
-| `environment` | 否 | 评测环境前置「已就绪」声明 |
+| `environment` | 否 | 题设环境声明（仅注入 prompt，不物化） |
 | `tripwire` | 否 | 标记为「故意诱错」样本，failed 时 diagnostic 不建议改 skill |
 
 完整 schema 见 [docs/specs/sample-design-spec.md](https://github.com/lizhiyao/oh-my-knowledge/blob/main/docs/specs/sample-design-spec.md)。

package/dist/authoring/generator.d.ts

@@ -1,6 +1,7 @@
-import type { Sample, ExecutorFn } from '../types/index.js';
+import type { ExecutorFn, Sample } from '../types/index.js';
 import type { ObservationInboxItem } from '../types/observability.js';
-interface GenerateSamplesOptions {
+export declare function sampleGenerationUsesMocks(executorName: string | undefined, noMock?: boolean): boolean;
+export interface GenerateSamplesOptions {
     skillContent: string;
     count?: number;
     model: string;
@@ -12,6 +13,8 @@ interface GenerateSamplesOptions {
     focus?: string;
     /** 不生成 mocks/mocksStrict，eval 时真实执行所有工具调用。 */
     noMock?: boolean;
+    /** Injectable executor for tests. Defaults to createExecutor(executorName). */
+    executor?: ExecutorFn;
 }
 /**
  * 拼出送给 LLM 的 user prompt。抽出来便于单测验证 focus 是否真的注入了。
@@ -26,7 +29,7 @@ export declare function buildSamplesPrompt({ skillContent, count, focus, noMock
     focus?: string;
     noMock?: boolean;
 }): string;
-export declare function generateSamples({ skillContent, count, model, executorName, focus, noMock }: GenerateSamplesOptions): Promise<{
+export declare function generateSamples({ skillContent, count, model, executorName, focus, noMock, executor: injectedExecutor, }: GenerateSamplesOptions): Promise<{
     samples: Sample[];
     costUSD: number;
 }>;
@@ -62,12 +65,16 @@ export declare function stratifyTraceSignals(items: TraceSignalItem[]): Stratifi
  * production failures. The trace text feeds the *generator* only — never the
  * judge prompt — so judge-prompt isolation is unaffected.
  */
-export declare function buildSamplesFromTracesPrompt(items: TraceSignalItem[], count?: number): string;
+export declare function buildSamplesFromTracesPrompt(items: TraceSignalItem[], count?: number, options?: {
+    noMock?: boolean;
+}): string;
 export interface GenerateSamplesFromTracesOptions {
     items: TraceSignalItem[];
     count?: number;
     model: string;
     executorName?: string;
+    /** 不生成 mocks。目标 executor 不支持时会自动启用。 */
+    noMock?: boolean;
     /** Injectable executor (tests). Defaults to createExecutor(executorName). */
     executor?: ExecutorFn;
 }
@@ -77,12 +84,15 @@ export interface GenerateSamplesFromTracesOptions {
  * stamps `provenance: 'production-trace'`. Output is meant to land in a review draft,
  * not the live dataset (the CLI enforces that).
  */
-export declare function generateSamplesFromTraces({ items, count, model, executorName, executor: injectedExecutor, }: GenerateSamplesFromTracesOptions): Promise<{
+export declare function generateSamplesFromTraces({ items, count, model, executorName, noMock, executor: injectedExecutor, }: GenerateSamplesFromTracesOptions): Promise<{
     samples: Sample[];
     costUSD: number;
 }>;
 export declare function sanitizeGeneratedSamples(samples: Sample[], opts?: {
     skillContent?: string;
+    mockless?: boolean;
+    migrateMocklessEnvironment?: boolean;
+    preserveMocklessEnvironment?: boolean;
 }): {
     stripped: string[];
 };

package/dist/authoring/generator.js

@@ -1,5 +1,7 @@
 import { createExecutor } from '../executors/index.js';
+import { executorSupportsSampleMocks } from '../executors/capabilities.js';
 import { DEFAULT_GATE_THRESHOLD } from '../eval-core/verdict.js';
+import { sampleMockReferenceKeys } from '../shared/sample-contract.js';
 const SYSTEM_PROMPT = `你是一个评测用例生成器。你的任务是根据用户提供的 skill（系统提示词）内容，生成高质量的评测用例。
 
 样本结构决策（必须先做）：先扫一遍 skill 内容判断它属于哪一类，按对应配比和数量生成。
@@ -195,14 +197,14 @@ const SYSTEM_PROMPT = `你是一个评测用例生成器。你的任务是根据
         tool_input_not_contains 或 tools_not_called。
   - { "type": "regex", "pattern": "...", "weight": 1 }
         ↑ 同 contains 限制:只用在固定格式字面量(如 SHA / UUID / 路径模板)。
-- environment: 可选,对象。**评测环境的"已就绪"声明**,LLM 看到后跳过环境探测直接进工作流。
+- environment: 可选,对象。**仅作 prompt 上下文的题设环境声明**,不会修改 PATH、创建文件或物化 fixture。
   字段:
     - cli_available: string[],已在 PATH 上的 CLI(如 ["node", "git", "code-host"])
     - files_available: string[],已存在的文件/脚本(如 ["~/.req-tool-api.json", "$SKILL_DIR/scripts/x.js"])
     - notes: string,自由文本兜底(如"DevAPI 凭证有效,工号 testuser001")
   原则:
-    凡是 skill 跑起来需要的环境(凭证文件 / 业务 CLI / 自带脚本 / API token 等),
-    都写到这里,而不是在 mock 里 mock 它们的探测命令。这让 mock 只关注业务调用本身。
+    只有用例明确把某项环境能力作为题设前提时才写。需要读取真实内容的文件不能放在
+    files_available 里冒充 fixture:应放进 sample.cwd 下的真实 fixture,或由 mock 返回内容。
 - mocksStrict: **必填且必须设为 true**(只要 sample 配了 mocks)。
   原因:mocksStrict=false 时,LLM 调到没匹配 mock 的命令会**透传到真 shell**,
   既可能真调外部接口产生副作用,也可能因二进制不存在(如 mcporter)报噪声错误污染评测信号。
@@ -223,8 +225,8 @@ const SYSTEM_PROMPT = `你是一个评测用例生成器。你的任务是根据
            - 对(✅):写一条宽 mock:\`{tool:"Bash", match:{command_glob:"ls *"},
              return:{stdout:"<模拟目录列表>", exit:0}}\` — \`command_glob\` 用 \`*\` 兜底各种
              ls 参数变体(\`ls\` / \`ls -la\` / \`ls -d\` / \`ls /xx\` 全命中)
-       (c) 单纯"已就绪"声明(凭证文件 / 业务 CLI 是否安装)还是走 \`environment\` 字段,
-           不需要 LLM 真调命令检查 — environment 字段就是告诉 LLM "这些不用检查"。
+       (c) 题设明确声明可用的凭证 / CLI 可写进 \`environment\`,让 LLM 不做可用性探测。
+           该字段只进 prompt,不会真的安装 CLI、创建凭证文件或改变 runtime。
        (d) **intent-level mock(文件搜索/读取类操作)** — LLM 搜代码时会自由选择 Bash grep、
            Grep 工具、Glob+Read 组合、甚至 Agent 子代理,逐个枚举工具写 mock 不可持续。
            正确做法:用 \`tool: "*"\` + \`input_contains: "关键词"\` 按意图匹配:
@@ -367,6 +369,37 @@ const SYSTEM_PROMPT = `你是一个评测用例生成器。你的任务是根据
 - 字符串字段（prompt / rubric / capability 等）内部如需引号，**必须用全角「」**而不是半角 \`""\`，避免漏转义破坏 JSON 解析
 - 例：错 → \`"prompt": "查询"Daily"标签..."\`（内部 \`"\` 未转义，JSON 解析失败）
        对 → \`"prompt": "查询「Daily」标签..."\`（全角引号，无转义压力）`;
+const MOCKLESS_SYSTEM_OVERRIDE = `
+
+## 目标执行器能力约束（最高优先级）
+
+目标执行器不支持可靠的工具调用拦截。本节覆盖上文所有要求 mocks、mock_hit、
+files_available 或正向工具调用断言的规则：
+
+- 不要输出 mocks、mocksStrict、environment 字段。
+- 不要输出 mock_hit、tools_called、tools_count_min、tool_input_contains、tool_output_contains。
+- 可以保留 tools_not_called、tools_count_max、tool_input_not_contains 这类负向安全约束。
+- 把必要的输入事实直接写进 context，把可判分结果写进 rubric 或输出内容断言。
+- 不要假装某个文件、CLI 或工具调用已经由 fixture 物化。`;
+function generationSystemPrompt(mockless) {
+    return mockless ? `${SYSTEM_PROMPT}${MOCKLESS_SYSTEM_OVERRIDE}` : SYSTEM_PROMPT;
+}
+const warnedAutoMocklessExecutors = new Set();
+function warnAutoMockless(executorName, noMock) {
+    if (noMock
+        || !executorName
+        || executorSupportsSampleMocks(executorName)
+        || warnedAutoMocklessExecutors.has(executorName))
+        return;
+    warnedAutoMocklessExecutors.add(executorName);
+    process.stderr.write(`[omk sample] 执行器「${executorName}」不支持工具调用拦截，`
+        + '已自动切换为无 mocks 用例。\n');
+}
+export function sampleGenerationUsesMocks(executorName, noMock = false) {
+    if (noMock)
+        return false;
+    return executorName ? executorSupportsSampleMocks(executorName) : true;
+}
 /**
  * 拼出送给 LLM 的 user prompt。抽出来便于单测验证 focus 是否真的注入了。
  *
@@ -390,9 +423,12 @@ ${skillContent}
 
 ${countLine}直接输出 JSON 数组。${focusBlock}${noMockBlock}`;
 }
-export async function generateSamples({ skillContent, count, model, executorName, focus, noMock }) {
-    const executor = createExecutor(executorName);
-    const prompt = buildSamplesPrompt({ skillContent, count, focus, noMock });
+export async function generateSamples({ skillContent, count, model, executorName, focus, noMock, executor: injectedExecutor, }) {
+    const executor = injectedExecutor ?? createExecutor(executorName);
+    const mockless = !sampleGenerationUsesMocks(executorName, noMock);
+    warnAutoMockless(executorName, noMock);
+    const prompt = buildSamplesPrompt({ skillContent, count, focus, noMock: mockless });
+    const system = generationSystemPrompt(mockless);
     // 生成场景比单次 eval 调用更重(LLM 要思考结构 + 输出大段 JSON),
     // 默认 120s 对长 skill + count >= 8 经常不够,这里用 5 分钟兜底。
     // lean=true 关掉 agent 工具循环 / skill 发现 — 生成只需要纯文本,不需要 Bash / Read 等工具。
@@ -405,7 +441,7 @@ export async function generateSamples({ skillContent, count, model, executorName
         const attemptPrompt = attempt === 1
             ? prompt
             : `${prompt}\n\n上一次输出解析失败:${lastErr}\n请严格按 JSON 规范输出(字符串内部用「」全角引号),只输出数组,不要包含其他文字。`;
-        const result = await executor({ model, system: SYSTEM_PROMPT, prompt: attemptPrompt, timeoutMs: 300_000, lean: true });
+        const result = await executor({ model, system, prompt: attemptPrompt, timeoutMs: 300_000, lean: true });
         totalCost += result.costUSD || 0;
         if (!result.ok) {
             lastErr = result.error || 'unknown error';
@@ -435,12 +471,15 @@ export async function generateSamples({ skillContent, count, model, executorName
             continue;
         }
         // 通过校验,跳出循环继续后续 sanitize
-        return await finalizeSamples(samples, totalCost, skillContent);
+        return await finalizeSamples(samples, totalCost, {
+            skillContent,
+            mockless,
+        });
     }
     // 不可达 (循环里所有出口都 throw 或 return),保留是为了 TS 类型推断
     throw new Error('unreachable');
 }
-async function finalizeSamples(samples, costUSD, skillContent) {
+async function finalizeSamples(samples, costUSD, options) {
     // Validate required fields + sanitize metadata enums *at generator boundary*
     // (see sanitizeGeneratedSamples). skillContent is passed so the function can
     // strip "脑补"-style fact assertions whose tool name has no literal mention
@@ -448,7 +487,10 @@ async function finalizeSamples(samples, costUSD, skillContent) {
     // the data-security-review v1-v5 regen series (generator kept producing
     // tool_input_contains "WebFetch:语雀URL" even after 7 prompt iterations,
     // because LLM's "URL → fetch" training prior overrides instructional text).
-    const { stripped } = sanitizeGeneratedSamples(samples, { skillContent });
+    const { stripped } = sanitizeGeneratedSamples(samples, {
+        ...options,
+        migrateMocklessEnvironment: true,
+    });
     if (stripped.length > 0) {
         process.stderr.write(`[omk sample] LLM-output 含 ${stripped.length} 个非法元数据/断言字段，已剥离避免污染：\n  - ${stripped.join('\n  - ')}\n`);
     }
@@ -492,7 +534,7 @@ export function stratifyTraceSignals(items) {
  * production failures. The trace text feeds the *generator* only — never the
  * judge prompt — so judge-prompt isolation is unaffected.
  */
-export function buildSamplesFromTracesPrompt(items, count) {
+export function buildSamplesFromTracesPrompt(items, count, options = {}) {
     // 先按频次分层（合并重复 + 算占比 + 降序），让模型按「占比」分配配额，而非每信号一刀切。
     const stratified = stratifyTraceSignals(items);
     const sections = stratified.map((it, i) => {
@@ -515,13 +557,16 @@ export function buildSamplesFromTracesPrompt(items, count) {
     const countLine = typeof count === 'number'
         ? `共生成约 ${count} 条评测用例，按各信号的「占比」分配配额（高频多、低频少），覆盖整体失败分布。`
         : '按各信号「占比」分配：高频信号多生成、低频少生成，覆盖整体失败分布。';
+    const mocklessBlock = options.noMock
+        ? '\n\n目标执行器不支持工具调用拦截：不要生成 mocks、mocksStrict、environment 或正向工具调用断言；把 trace 证据写入 context 和 rubric。'
+        : '';
     return `${TRACE_GEN_INSTRUCTIONS}
 
 ## 观测到的失败信号（共 ${stratified.length} 个，已按出现频次降序）
 
 ${sections}
 
-${countLine}直接输出 JSON 数组。`;
+${countLine}直接输出 JSON 数组。${mocklessBlock}`;
 }
 /** Build a sanitize context string from trace evidence so finalizeSamples keeps
  *  tool-name assertions that reference tools actually seen in the traces (an empty
@@ -539,14 +584,17 @@ function traceSanitizeContext(items) {
  * stamps `provenance: 'production-trace'`. Output is meant to land in a review draft,
  * not the live dataset (the CLI enforces that).
  */
-export async function generateSamplesFromTraces({ items, count, model, executorName, executor: injectedExecutor, }) {
+export async function generateSamplesFromTraces({ items, count, model, executorName, noMock, executor: injectedExecutor, }) {
     if (items.length === 0)
         return { samples: [], costUSD: 0 };
     if (!injectedExecutor && !executorName) {
         throw new Error('executorName is required when no executor is injected');
     }
     const executor = injectedExecutor ?? createExecutor(executorName);
-    const prompt = buildSamplesFromTracesPrompt(items, count);
+    const mockless = !sampleGenerationUsesMocks(executorName, noMock);
+    warnAutoMockless(executorName, noMock);
+    const prompt = buildSamplesFromTracesPrompt(items, count, { noMock: mockless });
+    const system = generationSystemPrompt(mockless);
     const sanitizeContext = traceSanitizeContext(items);
     const PROVENANCE = 'production-trace';
     const MAX_ATTEMPTS = 3;
@@ -556,7 +604,7 @@ export async function generateSamplesFromTraces({ items, count, model, executorN
         const attemptPrompt = attempt === 1
             ? prompt
             : `${prompt}\n\n上一次输出解析失败:${lastErr}\n请严格按 JSON 规范输出(字符串内部用「」全角引号),只输出数组,不要包含其他文字。`;
-        const result = await executor({ model, system: SYSTEM_PROMPT, prompt: attemptPrompt, timeoutMs: 300_000, lean: true });
+        const result = await executor({ model, system, prompt: attemptPrompt, timeoutMs: 300_000, lean: true });
         totalCost += result.costUSD || 0;
         if (!result.ok) {
             lastErr = result.error || 'unknown error';
@@ -593,7 +641,10 @@ export async function generateSamplesFromTraces({ items, count, model, executorN
         // Stamp provenance before sanitize so it survives (it's a valid enum value).
         for (const s of samples)
             s.provenance = PROVENANCE;
-        return await finalizeSamples(samples, totalCost, sanitizeContext);
+        return await finalizeSamples(samples, totalCost, {
+            skillContent: sanitizeContext,
+            mockless,
+        });
     }
     throw new Error('unreachable');
 }
@@ -656,6 +707,78 @@ const TEXT_VALUE_TYPES = new Set([
 const TOOL_POSITIVE_TYPES = new Set([
     'tool_input_contains', 'tool_output_contains', 'mock_hit',
 ]);
+const MOCKLESS_POSITIVE_TOOL_TYPES = new Set([
+    'mock_hit',
+    'tools_called',
+    'tools_count_min',
+    'tool_input_contains',
+    'tool_output_contains',
+]);
+function stripMocklessPositiveToolAssertions(assertions, label, stripped) {
+    const kept = [];
+    for (const [index, assertion] of assertions.entries()) {
+        const assertionLabel = `${label}[${index}]`;
+        if (assertion?.type === 'assert-set' && Array.isArray(assertion.children)) {
+            const children = stripMocklessPositiveToolAssertions(assertion.children, `${assertionLabel}.children`, stripped);
+            if (children.length === 0) {
+                stripped.push(`${assertionLabel}.assert-set（无 mock 模式下没有可执行子断言）`);
+                continue;
+            }
+            assertion.children = children;
+            kept.push(assertion);
+            continue;
+        }
+        if (MOCKLESS_POSITIVE_TOOL_TYPES.has(assertion?.type)) {
+            stripped.push(`${assertionLabel}.${assertion.type}（目标执行器不支持 mocks，正向工具证据不可复现）`);
+            continue;
+        }
+        kept.push(assertion);
+    }
+    return kept;
+}
+function stripInvalidMockHitAssertions(assertions, mockKeys, label, stripped) {
+    const kept = [];
+    for (const [index, assertion] of assertions.entries()) {
+        const assertionLabel = `${label}[${index}]`;
+        if (assertion?.type === 'assert-set' && Array.isArray(assertion.children)) {
+            const children = stripInvalidMockHitAssertions(assertion.children, mockKeys, `${assertionLabel}.children`, stripped);
+            if (children.length === 0) {
+                stripped.push(`${assertionLabel}.assert-set（没有可执行子断言）`);
+                continue;
+            }
+            assertion.children = children;
+            kept.push(assertion);
+            continue;
+        }
+        if (assertion?.type === 'mock_hit'
+            && typeof assertion.value === 'string'
+            && !mockKeys.has(assertion.value)) {
+            stripped.push(`${assertionLabel}.mock_hit（未引用实际 mock：${assertion.value}）`);
+            continue;
+        }
+        kept.push(assertion);
+    }
+    return kept;
+}
+function environmentAsPromptContext(environment) {
+    if (!environment || typeof environment !== 'object' || Array.isArray(environment)) {
+        return null;
+    }
+    const env = environment;
+    const lines = ['题设环境声明（仅作上下文，不会在 cwd 物化）：'];
+    if (Array.isArray(env.cli_available)
+        && env.cli_available.every((item) => typeof item === 'string' && item.length > 0)) {
+        lines.push(`- 可用 CLI：${env.cli_available.join('、')}`);
+    }
+    if (Array.isArray(env.files_available)
+        && env.files_available.every((item) => typeof item === 'string' && item.length > 0)) {
+        lines.push(`- 题设引用路径（未物化）：${env.files_available.join('、')}`);
+    }
+    if (typeof env.notes === 'string' && env.notes.trim()) {
+        lines.push(`- 说明：${env.notes.trim()}`);
+    }
+    return lines.length > 1 ? lines.join('\n') : null;
+}
 function isAsciiTokenLike(v) {
     if (typeof v !== 'string')
         return false;
@@ -719,6 +842,40 @@ export function sanitizeGeneratedSamples(samples, opts = {}) {
             stripped.push(`samples[${i}].tripwire (${typeof s.tripwire})`);
             delete s.tripwire;
         }
+        if (opts.mockless) {
+            if (Array.isArray(s.mocks) && s.mocks.length > 0) {
+                stripped.push(`samples[${i}].mocks（目标执行器不支持工具调用拦截）`);
+            }
+            delete s.mocks;
+            if (s.mocksStrict !== undefined) {
+                stripped.push(`samples[${i}].mocksStrict（无 mocks）`);
+            }
+            delete s.mocksStrict;
+            if (s.environment !== undefined && !opts.preserveMocklessEnvironment) {
+                const environmentContext = opts.migrateMocklessEnvironment
+                    ? environmentAsPromptContext(s.environment)
+                    : null;
+                if (environmentContext) {
+                    const existingContext = typeof s.context === 'string' ? s.context.trim() : '';
+                    s.context = existingContext
+                        ? `${existingContext}\n\n${environmentContext}`
+                        : environmentContext;
+                    stripped.push(`samples[${i}].environment（已迁移到题设 context，未物化）`);
+                }
+                else {
+                    stripped.push(`samples[${i}].environment（未物化的环境声明不能充当 fixture）`);
+                }
+            }
+            if (!opts.preserveMocklessEnvironment) {
+                delete s.environment;
+            }
+            if (Array.isArray(s.assertions)) {
+                s.assertions = stripMocklessPositiveToolAssertions(s.assertions, `samples[${i}].assertions`, stripped);
+                if (s.assertions.length === 0) {
+                    delete s.assertions;
+                }
+            }
+        }
         // assertions 校验:loader 会拒掉两类无效断言 — 在 generator boundary 提前 strip,
         // 避免落盘的 sample 跑不动:
         //   1. tools_called / tools_not_called 的 values 必须非空
@@ -843,6 +1000,12 @@ export function sanitizeGeneratedSamples(samples, opts = {}) {
                     delete s.mocks;
             }
         }
+        if (Array.isArray(s.assertions)) {
+            s.assertions = stripInvalidMockHitAssertions(s.assertions, sampleMockReferenceKeys(s.mocks), `samples[${i}].assertions`, stripped);
+            if (s.assertions.length === 0) {
+                delete s.assertions;
+            }
+        }
         // mocksStrict 兜底:有 mocks 时强制 true。
         // SYSTEM_PROMPT 已要求 LLM 必填,但偶尔 LLM 漏填 — 在 generator boundary 修掉,
         // 避免运行时 mock 未命中透传到真 shell(报 mcporter not found 等噪声错误)。

package/dist/authoring/sample-fixer.d.ts

@@ -34,6 +34,8 @@ export interface FixSamplesOptions {
     }>;
     model: string;
     maxAttemptsPerSample?: number;
+    /** Target executor cannot intercept tool calls; strip mocks and dependent positive evidence. */
+    mockless?: boolean;
 }
 export interface FixSamplesResult {
     samples: Record<string, unknown>[];

package/dist/authoring/sample-fixer.js

@@ -20,8 +20,17 @@ const FIX_SYSTEM_PROMPT = `你是一个评测用例修复专家。根据诊断
 输出一个 **JSON 数组**,包含所有待修复 sample（修改过的和原样保留的都要包含）。
 第一字符 \`[\`，最后 \`]\`，不要用 \`\`\`json\`\`\` 围栏，不要寒暄。
 如果判断某条是 LLM 行为问题不需要改,也原样放进数组。`;
-function sanitizeFixedSamples(samples, skillContent) {
-    sanitizeGeneratedSamples(samples, { skillContent });
+const MOCKLESS_FIX_OVERRIDE = `
+
+目标执行器不支持工具调用拦截。不得新增或保留 mocks、mocksStrict、
+mock_hit、tools_called、tools_count_min、tool_input_contains、tool_output_contains。
+environment 仅表示题设上下文，不得当作已物化 fixture；负向安全断言可以保留。`;
+function sanitizeFixedSamples(samples, skillContent, mockless) {
+    sanitizeGeneratedSamples(samples, {
+        skillContent,
+        mockless,
+        preserveMocklessEnvironment: true,
+    });
     return samples;
 }
 const FIXABLE_SAMPLE_FIELDS = new Set([
@@ -97,7 +106,7 @@ function parseFixedSamples(text) {
     return null;
 }
 export async function fixSamples(options) {
-    const { skillContent, samples, report, treatmentKey, executor, model, maxAttemptsPerSample = 2 } = options;
+    const { skillContent, samples, report, treatmentKey, executor, model, maxAttemptsPerSample = 2, mockless = false, } = options;
     const sampleMap = new Map(samples.map((s) => [s.sample_id, s]));
     // Collect all fixable samples into one batch
     const fixContexts = [];
@@ -185,7 +194,12 @@ ${sampleSections}
     let incurredCostUSD = 0;
     let incurredCostReported = false;
     try {
-        const result = await executor({ model, system: FIX_SYSTEM_PROMPT, prompt, timeoutMs: 300_000 });
+        const result = await executor({
+            model,
+            system: mockless ? `${FIX_SYSTEM_PROMPT}${MOCKLESS_FIX_OVERRIDE}` : FIX_SYSTEM_PROMPT,
+            prompt,
+            timeoutMs: 300_000,
+        });
         incurredCostUSD = result.costUSD;
         incurredCostReported = result.costReported !== false;
         if (!result.ok) {
@@ -226,7 +240,7 @@ ${sampleSections}
                 })),
             };
         }
-        const sanitizedFixedArr = sanitizeFixedSamples(fixedArr, skillContent);
+        const sanitizedFixedArr = sanitizeFixedSamples(fixedArr, skillContent, mockless);
         const outOfScope = sanitizedFixedArr.find((fixed) => {
             const sid = fixed.sample_id;
             const original = sampleMap.get(sid);

package/dist/cli/commands/sample.js

@@ -274,7 +274,7 @@ async function runSampleFix(args, flags, lang) {
         throw new CliExit(1);
     }
     process.stderr.write(lang === 'zh' ? `🔧 发现 ${sampleDesignCount} 条 sample_design 失败，开始修复...\n` : `🔧 Found ${sampleDesignCount} sample_design failure(s), fixing...\n`);
-    const { createExecutor } = await import('../../executors/index.js');
+    const { createExecutor, executorSupportsSampleMocks, } = await import('../../executors/index.js');
     const exec = createExecutor(executorName);
     const executorFn = async (opts) => {
         const result = await exec({
@@ -298,6 +298,7 @@ async function runSampleFix(args, flags, lang) {
         treatmentKey: treatmentName,
         executor: executorFn,
         model,
+        mockless: !executorSupportsSampleMocks(executorName),
     });
     let writtenFiles = [];
     if (result.fixedCount > 0) {
@@ -363,7 +364,13 @@ export async function runSampleFromTraces(flags, lang) {
         ? `🔭 发现 ${items.length} 个${flags.skill ? ` ${flags.skill} 的` : ''}失败信号，正在生成评测用例草稿...\n`
         : `🔭 Found ${items.length}${flags.skill ? ` ${flags.skill}` : ''} failure signal(s); generating regression-sample drafts...\n`);
     try {
-        const { samples, costUSD } = await generateSamplesFromTraces({ items, count, model, executorName });
+        const { samples, costUSD } = await generateSamplesFromTraces({
+            items,
+            count,
+            model,
+            executorName,
+            noMock: flags['no-mock'],
+        });
         const cost = costUSD > 0 ? ` $${costUSD.toFixed(4)}` : '';
         if (samples.length === 0) {
             // The model conservatively skipped every signal (noise / unreproducible). That's a
@@ -663,8 +670,8 @@ export default class Sample extends BaseCommand {
         }),
         'no-mock': Flags.boolean({
             description: bilingual({
-                zh: '不生成 mocks，eval 时所有工具调用真实执行。',
-                en: 'Skip mock generation; all tool calls execute for real during eval.',
+                zh: '不生成 mocks。执行器不支持工具拦截时会自动启用，避免产生必然失败的 mock_hit。',
+                en: 'Skip mocks. Automatically enabled when the executor cannot intercept tools, preventing impossible mock_hit assertions.',
             }),
             default: false,
         }),

package/dist/eval-core/evaluation-execution.js

@@ -258,8 +258,17 @@ export async function executeTasks({ tasks, executor, executorName, model, noJud
             }
         }
         let factCheck;
-        if (execResult.ok && execResult.output && task.cwd) {
-            factCheck = checkFacts(execResult.output, resolve(task.cwd));
+        if (execResult.ok && execResult.output) {
+            const sharedEvidence = {
+                ...(task._sample.context && { context: task._sample.context }),
+                ...(task._sample.environment?.files_available?.length && {
+                    declaredFiles: task._sample.environment.files_available,
+                }),
+                // Resolve exactly like the executor's cwd. samplesBaseDir is for bundle
+                // assets such as mocks, not for changing Sample.cwd path semantics.
+                ...(task._sample.cwd && { cwd: resolve(task._sample.cwd) }),
+            };
+            factCheck = checkFacts(execResult.output, sharedEvidence);
         }
         const sampleResults = ownRecordValue(results, task.sample_id)
             ?? setOwnRecordValue(results, task.sample_id, {});

package/dist/eval-core/fact-checker.d.ts

@@ -14,11 +14,24 @@ export interface FactCheckResult {
     totalCount: number;
     verifiedRate: number;
 }
+export interface FactCheckEvidence {
+    /** Shared sample fixture root. Arm-specific execution directories must not be used. */
+    cwd?: string;
+    /** Facts supplied to both arms in the sample context. */
+    context?: string;
+    /** Declarative fixture paths from sample.environment.files_available. */
+    declaredFiles?: string[];
+}
 /**
  * Extract file path claims from agent output text.
  */
 export declare function extractPathClaims(output: string): string[];
 /**
- * Check facts in agent output by verifying file paths exist in cwd.
+ * Check file-path facts against sample-level evidence shared by every arm.
+ *
+ * A string keeps the legacy direct-filesystem API for callers outside the
+ * evaluation pipeline. The pipeline passes structured evidence and deliberately
+ * excludes each artifact's execution cwd, because that directory is not a
+ * comparable source of truth across control and treatment arms.
  */
-export declare function checkFacts(output: string, cwd: string): FactCheckResult;
+export declare function checkFacts(output: string, evidence: string | FactCheckEvidence): FactCheckResult;

package/dist/eval-core/fact-checker.js

@@ -22,6 +22,10 @@ const IGNORE_PATTERNS = [
     /^\.git\//,
     /^index\.\w+$/,
 ];
+// Product/runtime names that happen to end in a supported source extension.
+const NON_PATH_REFERENCES = new Set([
+    'node.js',
+]);
 /**
  * Extract file path claims from agent output text.
  */
@@ -36,35 +40,86 @@ export function extractPathClaims(output) {
             path = path.replace(/[.,;:!?）)]+$/, '');
             // Clean trailing backtick/quote
             path = path.replace(/[`'"]+$/, '');
-            if (path.length > 3 && !IGNORE_PATTERNS.some((p) => p.test(path))) {
+            if (path.length > 3
+                && !NON_PATH_REFERENCES.has(path.toLowerCase())
+                && !IGNORE_PATTERNS.some((p) => p.test(path))) {
                 paths.add(path);
             }
         }
     }
     return [...paths];
 }
+function normalizeEvidencePath(path) {
+    return path
+        .trim()
+        .replace(/^['"`]+|['"`]+$/g, '')
+        .replace(/\\/g, '/')
+        .replace(/^(?:\$SKILL_DIR|~)\//, '')
+        .replace(/^\.\//, '')
+        .replace(/\/+$/, '');
+}
+function isRootedEvidencePath(path) {
+    return path.startsWith('/') || /^[a-zA-Z]:\//.test(path);
+}
+function refersToSamePath(left, right) {
+    const a = normalizeEvidencePath(left);
+    const b = normalizeEvidencePath(right);
+    return a === b
+        || (isRootedEvidencePath(a) && !isRootedEvidencePath(b) && a.endsWith(`/${b}`))
+        || (isRootedEvidencePath(b) && !isRootedEvidencePath(a) && b.endsWith(`/${a}`));
+}
 /**
- * Check facts in agent output by verifying file paths exist in cwd.
+ * Check file-path facts against sample-level evidence shared by every arm.
+ *
+ * A string keeps the legacy direct-filesystem API for callers outside the
+ * evaluation pipeline. The pipeline passes structured evidence and deliberately
+ * excludes each artifact's execution cwd, because that directory is not a
+ * comparable source of truth across control and treatment arms.
  */
-export function checkFacts(output, cwd) {
+export function checkFacts(output, evidence) {
     const pathClaims = extractPathClaims(output);
-    const root = resolve(cwd);
-    const claims = pathClaims.map((path) => {
-        const fullPath = resolve(root, path);
-        const relativePath = relative(root, fullPath);
-        const insideCwd = relativePath === ''
-            || (!relativePath.startsWith('..') && !isAbsolute(relativePath));
-        const exists = insideCwd && existsSync(fullPath);
-        return {
-            type: 'file-path',
-            value: path,
-            verified: exists,
-            ...(!exists && {
-                evidence: insideCwd
-                    ? `${fullPath} not found`
-                    : `${path} is outside the evaluation cwd`,
-            }),
-        };
+    const sources = typeof evidence === 'string'
+        ? { cwd: evidence }
+        : evidence;
+    const root = sources.cwd ? resolve(sources.cwd) : null;
+    const contextClaims = sources.context
+        ? extractPathClaims(sources.context)
+        : [];
+    const declaredFiles = sources.declaredFiles ?? [];
+    const claims = pathClaims.flatMap((path) => {
+        if (declaredFiles.some((declared) => refersToSamePath(path, declared))) {
+            return [{
+                    type: 'file-path',
+                    value: path,
+                    verified: true,
+                    evidence: 'source=context(sample.environment.files_available)',
+                }];
+        }
+        if (root) {
+            const fullPath = resolve(root, path);
+            const relativePath = relative(root, fullPath);
+            const insideCwd = relativePath === ''
+                || (!relativePath.startsWith('..') && !isAbsolute(relativePath));
+            const exists = insideCwd && existsSync(fullPath);
+            return [{
+                    type: 'file-path',
+                    value: path,
+                    verified: exists,
+                    evidence: insideCwd
+                        ? `source=runtime-filesystem; ${fullPath} ${exists ? 'exists' : 'not found'}`
+                        : `source=runtime-filesystem; ${path} is outside the evaluation cwd`,
+                }];
+        }
+        if (contextClaims.some((contextPath) => refersToSamePath(path, contextPath))) {
+            return [{
+                    type: 'file-path',
+                    value: path,
+                    verified: true,
+                    evidence: 'source=context(sample.context)',
+                }];
+        }
+        // Without a shared fixture, absence from context is unknown rather than false.
+        return [];
     });
     const verifiedCount = claims.filter((c) => c.verified).length;
     const totalCount = claims.length;

package/dist/eval-core/mock-hook.cjs

@@ -92,8 +92,47 @@ function anyStringContains(obj, needle) {
   return false;
 }
 
+const BUILTIN_TOOL_ALIASES = {
+  bash: 'Bash',
+  shell: 'Bash',
+  exec_command: 'Bash',
+  command_execution: 'Bash',
+  read: 'Read',
+  file_read: 'Read',
+  grep: 'Grep',
+  edit: 'Edit',
+  apply_patch: 'Edit',
+  file_change: 'Edit',
+  write: 'Write',
+  file_write: 'Write',
+  view_image: 'ViewImage',
+  viewimage: 'ViewImage',
+  write_stdin: 'WriteStdin',
+  writestdin: 'WriteStdin',
+  web_search: 'WebSearch',
+  websearch: 'WebSearch',
+};
+
+function canonicalToolName(name) {
+  const sourceName = String(name);
+  const builtin = BUILTIN_TOOL_ALIASES[sourceName.toLowerCase()];
+  if (builtin) return builtin;
+  const parts = sourceName.split('__').filter(Boolean);
+  if (parts[0] === 'mcp' && parts.length > 2) {
+    const providerParts = parts.slice(1, -1);
+    if (providerParts[0] === 'codex_apps' && providerParts.length > 1) providerParts.shift();
+    return providerParts.join('.') + '.' + parts[parts.length - 1];
+  }
+  return sourceName;
+}
+
+function toolIdentityMatches(expectedName, runtimeName) {
+  return expectedName === runtimeName
+    || canonicalToolName(expectedName) === canonicalToolName(runtimeName);
+}
+
 function isMockHit(mock, toolName, toolInput) {
-  if (mock.tool !== '*' && mock.tool !== toolName) return false;
+  if (mock.tool !== '*' && !toolIdentityMatches(mock.tool, toolName)) return false;
   const m = mock.match;
   if (!m) return true;
   const ti = toolInput || {};

package/dist/eval-core/mocks-runtime.js

@@ -18,6 +18,7 @@ import { homedir, tmpdir } from 'node:os';
 import { dirname, isAbsolute, join, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { incrementRecordCount, setOwnRecordValue } from '../shared/record-count.js';
+import { toolIdentityMatches } from '../shared/tool-identity.js';
 // ─── Match logic ────────────────────────────────────────────────────────────
 function expandHome(p) {
     if (p.startsWith('~/'))
@@ -112,7 +113,7 @@ function anyStringContains(obj, needle) {
 }
 /** 单条 mock 是否命中给定 tool 调用。 */
 export function isMockHit(mock, toolName, toolInput) {
-    if (mock.tool !== '*' && mock.tool !== toolName)
+    if (mock.tool !== '*' && !toolIdentityMatches(mock.tool, toolName))
         return false;
     const m = mock.match;
     if (!m)
@@ -414,8 +415,44 @@ function anyStringContains(obj, needle) {
   if (typeof obj === 'object' && obj !== null) return Object.values(obj).some((v) => anyStringContains(v, needle));
   return false;
 }
+const BUILTIN_TOOL_ALIASES = {
+  bash: 'Bash',
+  shell: 'Bash',
+  exec_command: 'Bash',
+  command_execution: 'Bash',
+  read: 'Read',
+  file_read: 'Read',
+  grep: 'Grep',
+  edit: 'Edit',
+  apply_patch: 'Edit',
+  file_change: 'Edit',
+  write: 'Write',
+  file_write: 'Write',
+  view_image: 'ViewImage',
+  viewimage: 'ViewImage',
+  write_stdin: 'WriteStdin',
+  writestdin: 'WriteStdin',
+  web_search: 'WebSearch',
+  websearch: 'WebSearch',
+};
+function canonicalToolName(name) {
+  const sourceName = String(name);
+  const builtin = BUILTIN_TOOL_ALIASES[sourceName.toLowerCase()];
+  if (builtin) return builtin;
+  const parts = sourceName.split('__').filter(Boolean);
+  if (parts[0] === 'mcp' && parts.length > 2) {
+    const providerParts = parts.slice(1, -1);
+    if (providerParts[0] === 'codex_apps' && providerParts.length > 1) providerParts.shift();
+    return providerParts.join('.') + '.' + parts[parts.length - 1];
+  }
+  return sourceName;
+}
+function toolIdentityMatches(expectedName, runtimeName) {
+  return expectedName === runtimeName
+    || canonicalToolName(expectedName) === canonicalToolName(runtimeName);
+}
 function isMockHit(mock, toolName, toolInput) {
-  if (mock.tool !== '*' && mock.tool !== toolName) return false;
+  if (mock.tool !== '*' && !toolIdentityMatches(mock.tool, toolName)) return false;
   const m = mock.match;
   if (!m) return true;
   const ti = toolInput || {};

package/dist/eval-core/task-planner.d.ts

@@ -1,8 +1,8 @@
 import type { Artifact, Sample, SampleEnvironment, Task } from '../types/index.js';
 /**
  * 把 sample.environment 渲染成自然语言段落,放在用户 prompt 前。
- * 让 LLM 读到"环境已就绪",跳过 Glob / find / which / Read 这些环境探测,
- * 直接进入 skill 描述的工作流 — 评测信号纯,mock 设计也简化(不用 mock 探测命令)。
+ * 这是题设上下文,不会修改 PATH、物化文件或改变 runtime;LLM 可据此跳过
+ * which / test -f 等可用性探测,直接进入 skill 描述的工作流。
  *
  * 输出 null 表示 sample 没声明 environment,prompt 不变。
  */

package/dist/eval-core/task-planner.js

@@ -1,7 +1,7 @@
 /**
  * 把 sample.environment 渲染成自然语言段落,放在用户 prompt 前。
- * 让 LLM 读到"环境已就绪",跳过 Glob / find / which / Read 这些环境探测,
- * 直接进入 skill 描述的工作流 — 评测信号纯,mock 设计也简化(不用 mock 探测命令)。
+ * 这是题设上下文,不会修改 PATH、物化文件或改变 runtime;LLM 可据此跳过
+ * which / test -f 等可用性探测,直接进入 skill 描述的工作流。
  *
  * 输出 null 表示 sample 没声明 environment,prompt 不变。
  */
@@ -10,26 +10,26 @@ export function renderEnvironmentSection(env) {
         return null;
     const lines = [];
     if (env.cli_available && env.cli_available.length > 0) {
-        lines.push('- 已安装 CLI(已在 PATH,无需 `which` / `command -v` / `type` 探测):');
+        lines.push('- 题设声明可用的 CLI（仅作上下文，不修改 PATH）：');
         for (const c of env.cli_available)
             lines.push(`  - \`${c}\``);
     }
     if (env.files_available && env.files_available.length > 0) {
-        lines.push('- 已存在文件(无需 Glob / `Read` / `test -f` 探测):');
+        lines.push('- 题设引用的文件路径（仅作上下文，不会在 cwd 物化）：');
         for (const f of env.files_available)
             lines.push(`  - \`${f}\``);
     }
     if (env.notes && env.notes.trim()) {
-        lines.push(`- 备注:${env.notes.trim()}`);
+        lines.push(`- 备注：${env.notes.trim()}`);
     }
     if (lines.length === 0)
         return null;
     return [
-        '## 评测环境前置(已就绪,无需探测)',
+        '## 题设环境声明（仅作上下文）',
         '',
         ...lines,
         '',
-        '请直接进入 skill 描述的主流程,**不要做环境检查 / `Glob` / `find` / `which` / `test -f` 等探测**。',
+        '请按以上题设进入 skill 描述的主流程，**不要额外做 `find` / `which` / `test -f` 等可用性探测**。这些声明不会自动创建文件或修改 runtime 环境。',
     ].join('\n');
 }
 export function buildTasks(samples, variants, skills) {

package/dist/eval-workflows/evaluation-pipeline.js

@@ -28,6 +28,7 @@ import { finalizeSuccessfulRun, initializeEvaluationRunState, persistFailedJob,
 import { finalizeEvaluationReport } from './evaluation-pipeline/report-finalize.js';
 import { emitIsolationWarnings, emitPowerWarnings } from './evaluation-pipeline/preflight-warnings.js';
 import { ownRecordValue, setOwnRecordValue } from '../shared/record-count.js';
+import { assertSamplesCompatibleWithExecutor } from '../executors/capabilities.js';
 // 兼容 re-export:测试与 run-evaluation.ts 动态 import 仍打 evaluation-pipeline.js
 export { buildPowerWarnings, buildIsolationWarnings } from './evaluation-pipeline/preflight-warnings.js';
 export { _computeTestSetHashForTest } from './evaluation-pipeline/test-set-hash.js';
@@ -35,6 +36,7 @@ export async function executeEvaluationPipeline({ samplesPath, samplesBaseDir, s
 // requires 现在由 runEvaluation 上游传给 doctor 处理; eval-pipeline 不再用
 // 但保留接口字段,避免破 programmatic API (类型层面接收, 内部忽略)
 requires: _requires, layeredStats = false, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias = true, budget, strictBaseline, runId, lang = 'zh', effort, noDiagnostic, }) {
+    assertSamplesCompatibleWithExecutor(samples, executorName, lang);
     const variantNames = artifacts.map((artifact) => artifact.name);
     const runState = await initializeEvaluationRunState({
         samplesPath,

package/dist/eval-workflows/run-evaluation.js

@@ -1,6 +1,6 @@
 import { resolve } from 'node:path';
 import { DEFAULT_OUTPUT_DIR, persistReport } from '../eval-core/evaluation-reporting.js';
-import { createExecutor } from '../executors/index.js';
+import { assertSamplesCompatibleWithExecutor, createExecutor, } from '../executors/index.js';
 import { discoverBatchSkills } from '../inputs/skill-loader.js';
 import { confidenceInterval, tTest, effectSize } from '../eval-core/statistics.js';
 import { executeBatchEvaluationRuns, buildBatchVariantSpecs } from './batch-evaluation-workflow.js';
@@ -26,6 +26,7 @@ export async function runEvaluation({ samplesPath, skillDir, variantSpecs = [],
         mcpConfig,
         strictBaseline,
     });
+    assertSamplesCompatibleWithExecutor(samples, executorName, lang);
     // doctor 强制门禁: skill 静态结构 + 元数据 + 依赖 + 用例契约。
     // 在 dryRun 分支之前跑, 让 dry-run 也得到 doctor 覆盖(保护 garbage-in 的 verdict)。
     // 默认强制启用; --skip-doctor 提供 escape hatch,典型场景是评测环境用 mock/stub

package/dist/executors/capabilities.d.ts

@@ -0,0 +1,15 @@
+import type { ExecutorFn, ExecutorInput, Sample } from '../types/index.js';
+export type SampleMockSupport = 'native-hooks' | 'delegated-script' | 'unsupported';
+export interface ExecutorCapabilities {
+    sampleMocks: SampleMockSupport;
+}
+/**
+ * Custom script executors receive the OMK_MOCK_* protocol environment and own
+ * the final adapter. Built-ins are explicit so unsupported runtimes can never
+ * silently turn mock assertions into model failures.
+ */
+export declare function getExecutorCapabilities(executorName: string): ExecutorCapabilities;
+export declare function executorSupportsSampleMocks(executorName: string): boolean;
+export declare function assertSamplesCompatibleWithExecutor(samples: Sample[], executorName: string, lang?: 'zh' | 'en'): void;
+export declare function assertExecutorInputCapabilities(executorName: string, input: ExecutorInput): void;
+export declare function enforceExecutorCapabilities(executorName: string, executor: ExecutorFn): ExecutorFn;

package/dist/executors/capabilities.js

@@ -0,0 +1,64 @@
+const BUILTIN_CAPABILITIES = {
+    claude: { sampleMocks: 'native-hooks' },
+    'claude-sdk': { sampleMocks: 'native-hooks' },
+    codex: { sampleMocks: 'unsupported' },
+    'codex-sdk': { sampleMocks: 'unsupported' },
+    gemini: { sampleMocks: 'unsupported' },
+    'anthropic-api': { sampleMocks: 'unsupported' },
+    'openai-api': { sampleMocks: 'unsupported' },
+};
+/**
+ * Custom script executors receive the OMK_MOCK_* protocol environment and own
+ * the final adapter. Built-ins are explicit so unsupported runtimes can never
+ * silently turn mock assertions into model failures.
+ */
+export function getExecutorCapabilities(executorName) {
+    return BUILTIN_CAPABILITIES[executorName]
+        ?? { sampleMocks: 'delegated-script' };
+}
+export function executorSupportsSampleMocks(executorName) {
+    return getExecutorCapabilities(executorName).sampleMocks !== 'unsupported';
+}
+function unsupportedMocksMessage(executorName, sampleIds, lang) {
+    const ids = sampleIds.slice(0, 8).join(', ');
+    const overflow = sampleIds.length > 8
+        ? lang === 'zh'
+            ? ` 等 ${sampleIds.length} 条`
+            : ` and ${sampleIds.length - 8} more`
+        : '';
+    if (lang === 'en') {
+        return `Executor "${executorName}" does not support Sample.mocks tool interception. `
+            + `Continuing would make mock_hit assertions structurally impossible and create false evidence. `
+            + `Affected samples: ${ids}${overflow}. `
+            + 'Regenerate them with "omk sample --no-mock", remove mocks/mock_hit, '
+            + 'or evaluate with claude/claude-sdk.';
+    }
+    return `执行器「${executorName}」不支持 Sample.mocks 工具拦截。`
+        + `继续运行会让 mock_hit 在结构上必然失败并产生伪证据。`
+        + `受影响用例：${ids}${overflow}。`
+        + '请用「omk sample --no-mock」重新生成、删除 mocks/mock_hit，'
+        + '或改用 claude/claude-sdk 评测。';
+}
+export function assertSamplesCompatibleWithExecutor(samples, executorName, lang = 'zh') {
+    if (executorSupportsSampleMocks(executorName))
+        return;
+    const affected = samples
+        .filter((sample) => Array.isArray(sample.mocks) && sample.mocks.length > 0)
+        .map((sample) => sample.sample_id);
+    if (affected.length === 0)
+        return;
+    throw new Error(unsupportedMocksMessage(executorName, affected, lang));
+}
+export function assertExecutorInputCapabilities(executorName, input) {
+    if (executorSupportsSampleMocks(executorName)
+        || !Array.isArray(input.mocks)
+        || input.mocks.length === 0)
+        return;
+    throw new Error(unsupportedMocksMessage(executorName, ['<programmatic-input>'], 'zh'));
+}
+export function enforceExecutorCapabilities(executorName, executor) {
+    return async (input) => {
+        assertExecutorInputCapabilities(executorName, input);
+        return executor(input);
+    };
+}

package/dist/executors/index.d.ts

@@ -2,4 +2,5 @@ import type { ExecutorFn } from '../types/index.js';
 import { extractAgentTrace } from './claude-sdk-trace.js';
 import { createScriptExecutor } from './script.js';
 export { extractAgentTrace, createScriptExecutor };
+export { assertExecutorInputCapabilities, assertSamplesCompatibleWithExecutor, executorSupportsSampleMocks, getExecutorCapabilities, } from './capabilities.js';
 export declare function createExecutor(name: string): ExecutorFn;

package/dist/executors/index.js

@@ -7,6 +7,7 @@ import { codexSdkExecutor } from './codex-sdk.js';
 import { geminiExecutor } from './gemini.js';
 import { openAiApiExecutor } from './openai-api.js';
 import { createScriptExecutor } from './script.js';
+import { enforceExecutorCapabilities } from './capabilities.js';
 // 命名一致性:provider HTTP 路径统一用 `<vendor>-api`(`anthropic-api` / `openai-api`),
 // vendor coding agent CLI 用 vendor 名(`claude` / `codex`)。`openai` 这个不带 -api 后缀的
 // 旧 alias 历史上指 openai-cli 子进程实现,删除后不再设别名 — 用 `--executor openai-api`。
@@ -20,9 +21,11 @@ const EXECUTOR_REGISTRY = {
     'openai-api': openAiApiExecutor,
 };
 export { extractAgentTrace, createScriptExecutor };
+export { assertExecutorInputCapabilities, assertSamplesCompatibleWithExecutor, executorSupportsSampleMocks, getExecutorCapabilities, } from './capabilities.js';
 export function createExecutor(name) {
     if (name.trim().length === 0) {
         throw new Error('executor name or script command is required');
     }
-    return EXECUTOR_REGISTRY[name] || createScriptExecutor(name);
+    const executor = EXECUTOR_REGISTRY[name] || createScriptExecutor(name);
+    return enforceExecutorCapabilities(name, executor);
 }

package/dist/server/skill-insights.js

@@ -250,7 +250,7 @@ function detectSkillDocGap(doctor, evalReport, observe, variantName) {
     const recs = [];
     if (depRule) {
         recs.push({
-            action: `在 sample.environment.files_available 加上文件路径,告诉 LLM "这些文件已就绪,无需探测"`,
+            action: '把已知路径写进 sample.context；只有纯题设前提才放 environment.files_available，且不要把它当作已物化 fixture',
             priority: severity,
             patch: {
                 target: 'sample-environment',

package/dist/shared/sample-contract.d.ts

@@ -1,3 +1,4 @@
 export declare function assertionContractValidationError(value: unknown, depth?: number, insideAssertSet?: boolean): string | undefined;
+export declare function sampleMockReferenceKeys(value: unknown): ReadonlySet<string>;
 export declare function dependencyRequirementsValidationError(value: unknown): string | undefined;
 export declare function sampleContractValidationError(value: unknown, expectedId?: string): string | undefined;

package/dist/shared/sample-contract.js

@@ -219,6 +219,38 @@ function mockValidationError(value) {
         return '"match.input" must be a JSON object when present';
     return undefined;
 }
+export function sampleMockReferenceKeys(value) {
+    const keys = new Set();
+    if (!Array.isArray(value))
+        return keys;
+    const countByTool = new Map();
+    for (const mock of value) {
+        if (!isRecord(mock) || !isNonEmptyString(mock.tool))
+            continue;
+        const ordinal = (countByTool.get(mock.tool) ?? 0) + 1;
+        countByTool.set(mock.tool, ordinal);
+        keys.add(`${mock.tool}:${ordinal}`);
+    }
+    return keys;
+}
+function mockHitReferenceValidationError(assertions, mockKeys) {
+    if (!Array.isArray(assertions))
+        return undefined;
+    for (const assertion of assertions) {
+        if (!isRecord(assertion))
+            continue;
+        if (assertion.type === 'mock_hit'
+            && typeof assertion.value === 'string'
+            && !mockKeys.has(assertion.value)) {
+            const available = mockKeys.size > 0 ? [...mockKeys].join(', ') : '(none)';
+            return `"mock_hit" references missing mock ${JSON.stringify(assertion.value)}; available mock keys: ${available}`;
+        }
+        const childError = mockHitReferenceValidationError(assertion.children, mockKeys);
+        if (childError)
+            return childError;
+    }
+    return undefined;
+}
 export function dependencyRequirementsValidationError(value) {
     if (!isRecord(value))
         return '"requires" must be an object';
@@ -276,6 +308,9 @@ export function sampleContractValidationError(value, expectedId) {
                 return `"mocks[${index}]": ${error}`;
         }
     }
+    const mockHitError = mockHitReferenceValidationError(value.assertions, sampleMockReferenceKeys(value.mocks));
+    if (mockHitError)
+        return mockHitError;
     if (value.mocksStrict !== undefined && typeof value.mocksStrict !== 'boolean') {
         return '"mocksStrict" must be boolean when present';
     }

package/dist/shared/tool-identity.d.ts

@@ -19,3 +19,11 @@ export interface ToolIdentityInput {
  * retaining source identity for audit and future protocol migrations.
  */
 export declare function normalizeToolIdentity(input: ToolIdentityInput): NormalizedToolIdentity;
+/**
+ * Match a source-neutral tool identity against a runtime-native tool name.
+ *
+ * Exact matching remains first for legacy/custom tools. Normalization then lets
+ * one mock identity work across adapters such as `Bash` ↔ `exec_command` and
+ * `Edit` ↔ `apply_patch`.
+ */
+export declare function toolIdentityMatches(expectedName: string, runtimeName: string): boolean;

package/dist/shared/tool-identity.js

@@ -62,6 +62,19 @@ export function normalizeToolIdentity(input) {
         ...(sourceName !== name ? { sourceName } : {}),
     };
 }
+/**
+ * Match a source-neutral tool identity against a runtime-native tool name.
+ *
+ * Exact matching remains first for legacy/custom tools. Normalization then lets
+ * one mock identity work across adapters such as `Bash` ↔ `exec_command` and
+ * `Edit` ↔ `apply_patch`.
+ */
+export function toolIdentityMatches(expectedName, runtimeName) {
+    if (expectedName === runtimeName)
+        return true;
+    return normalizeToolIdentity({ sourceName: expectedName }).name
+        === normalizeToolIdentity({ sourceName: runtimeName }).name;
+}
 function inferredMcpNamespace(sourceName) {
     const parts = sourceName.split('__').filter(Boolean);
     return parts[0] === 'mcp' && parts.length > 2

package/dist/types/eval.d.ts

@@ -35,12 +35,12 @@ export interface SampleCoverageTarget {
     /** 稳定引用：路径或 ID。reference/script 用 skill 根相对路径，workflow_node 用 workflowId.nodeId。 */
     ref: string;
 }
-/** Sample 评测环境前置:声明性"已就绪"清单,LLM 看到后跳过环境探测,直接进入工作流。
- *  类比 unit test 的 fixture / setup —— 评测是测 skill 工作流,不是测环境探测能力。 */
+/** Sample 题设环境声明。仅注入 prompt,不会修改 PATH、物化文件或改变 runtime。 */
 export interface SampleEnvironment {
-    /** 假定已在 PATH 上的 CLI,LLM 不再 which / find / type / command -v 探测。 */
+    /** 题设声明可用的 CLI,LLM 不再 which / find / type / command -v 探测。 */
     cli_available?: string[];
-    /** 假定存在的文件/脚本(支持 ~ / $SKILL_DIR / 绝对路径),LLM 不再 Glob / Read / test -f 探测。 */
+    /** 题设声明存在的文件/脚本(支持 ~ / $SKILL_DIR / 绝对路径),LLM 不再 Glob / Read / test -f 探测。
+     *  这是 prompt context,不会在 cwd 物化文件;需要真实读取时必须提供 sample.cwd 中的 fixture。 */
     files_available?: string[];
     /** 自由文本兜底,场景特殊说明(如"凭证已配""设备 SN xxx 已租"等)。 */
     notes?: string;
@@ -80,7 +80,8 @@ export type MockReturn = {
 } | string;
 /** 单条 Mock 规则。runtime 拦到匹配的 tool 调用即返回 mocked 结果,不放出去。 */
 export interface Mock {
-    /** 拦截的工具名,如 "Read" / "Bash" / "WebFetch" / "Edit" / "Write" / "Grep" / "Glob"。
+    /** source-neutral 工具身份,如 "Read" / "Bash" / "WebFetch" / "Edit" / "Write" / "Grep" / "Glob"。
+     *  executor adapter 会把 runtime-native 名称（如 exec_command / apply_patch）映射后匹配。
      *  特殊值 `"*"`:通配,匹配任何工具名(配合 match.input_contains 做 intent-level mock)。 */
     tool: string;
     /** 命中规则。所有字段 AND,字段未填即不限制。 */
@@ -136,7 +137,7 @@ export interface Sample {
      *  - true:未命中即 deny(防意外真调外部接口/CLI/MCP/写状态)。
      *  全 mock 评测场景建议 true,部分 mock 探索场景留 false。 */
     mocksStrict?: boolean;
-    /** 环境前置:声明性"已就绪"清单,LLM 跳过探测直接干活。详见 SampleEnvironment。 */
+    /** 题设环境声明,仅作 prompt 上下文。详见 SampleEnvironment。 */
     environment?: SampleEnvironment;
     [key: string]: unknown;
 }

package/dist/types/executor.d.ts

@@ -87,8 +87,9 @@ export interface ExecutorInput {
      *  - claude-sdk:转 in-process HookCallback 装到 SDK options.hooks.PreToolUse
      *  - claude-cli:物化为临时 settings.json + on-disk hook 脚本,跑完清理
      *  - script(自定义脚本):同样物化临时 settings,通过 env(OMK_MOCK_SETTINGS_FILE /
-     *    OMK_MOCK_MCP_CONFIG_FILE / OMK_MOCKS_FILE)暴露给脚本;脚本若包 Claude Code 兼容
-     *    CLI 可透传 --settings 复用同一 mock hook,否则忽略(不支持的 CLI 静默无 mock) */
+     *    OMK_MOCK_MCP_CONFIG_FILE / OMK_MOCKS_FILE)暴露给脚本;脚本负责消费该协议
+     *  - codex / codex-sdk / gemini / *-api:不支持,executor capability gate 会拒绝,
+     *    绝不静默忽略后把 mock_hit 记成模型失败 */
     mocks?: import('./eval.js').Mock[];
     /** 解析 mock.return_file 的相对路径锚点(默认 sample 文件所在目录)。 */
     mocksBaseDir?: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "oh-my-knowledge",
-  "version": "0.51.1",
+  "version": "0.51.2",
   "packageManager": "yarn@4.16.0",
   "description": "Evaluation framework for LLM knowledge inputs — prompts, RAG corpora, skills, agent workflows. Fix the model, vary the artifact. Built-in statistical rigor: bootstrap CI, Krippendorff α, length-debias, saturation curves.",
   "type": "module",
@@ -96,11 +96,11 @@
   "license": "MIT",
   "dependencies": {
     "@anthropic-ai/claude-agent-sdk": "^0.3.143",
-    "@anthropic-ai/sdk": "^0.112.3",
+    "@anthropic-ai/sdk": "^0.115.0",
     "@inquirer/prompts": "^8.4.3",
     "@modelcontextprotocol/sdk": "^1.29.0",
     "@oclif/core": "^4",
-    "@openai/codex-sdk": "0.144.6",
+    "@openai/codex-sdk": "0.145.0",
     "ajv": "^8.18.0",
     "chart.js": "^4.5.1",
     "es-module-lexer": "^2.0.0",
@@ -116,7 +116,7 @@
     "@types/node": "^25.5.0",
     "eslint": "^10.1.0",
     "husky": "^9.1.7",
-    "lint-staged": "17.1.0",
+    "lint-staged": "17.2.0",
     "npm-run-all2": "^9.0.1",
     "typescript": "^6.0.2",
     "typescript-eslint": "^8.58.0",