oh-my-knowledge 0.51.0 → 0.51.2

package/dist/assets/agent-skills/omk/SKILL.md

@@ -123,6 +123,8 @@ omk sample skills/my-skill/SKILL.md --focus "重点覆盖搜索失败 / 权限
 omk sample --batch
 ```
 
+目标执行器不支持工具拦截时，`omk sample` 会自动生成无 mock 用例。当前 `codex` / `codex-sdk` 属于这种情况；不要手工补 `mocks` 或 `mock_hit`。已有 mocks 用例会被 `omk eval` 在模型调用前拒绝，避免把执行器能力缺口误判成模型失败。`environment.files_available` 只提供题设上下文，不会在 `cwd` 物化文件。
+
 输出位置：目录 skill（`<skill>/SKILL.md`）→ `<skill>/.omk/samples.json`（标准）；扁平 `.md` 单次生成 → 当前目录 `eval-samples.json`（项目级兜底）；扁平 `.md` 的 `--batch` 兼容生成 `<skill-dir>/<name>.eval-samples.json`。
 
 ### 观测真实使用

package/dist/assets/agent-skills/omk/references/commands.md

@@ -602,7 +602,7 @@ omk sample [skillPath] [flags]
 - `--from-traces` `boolean`:from-traces 模式：从 observe inbox 的失败信号回流生成评测用例草稿（provenance: production-trace），落草稿待人工 review。
 - `--lang` `option` (默认 `zh`):输出语言 zh|en，优先级 CLI > OMK_LANG env > zh。
 - `--model` `option`:生成 LLM model 名。Codex 自动读取本机配置；也可用 OMK_MODEL 设置环境偏好。
-- `--no-mock` `boolean`:不生成 mocks，eval 时所有工具调用真实执行。
+- `--no-mock` `boolean`:不生成 mocks。执行器不支持工具拦截时会自动启用，避免产生必然失败的 mock_hit。
 - `--observations-dir` `option`:observe inbox 目录（from-traces 模式用），默认项目 .omk/observe-inbox。
 - `--reports-dir` `option`:报告目录（fix 模式用），默认 ~/.oh-my-knowledge/reports。
 - `--skill` `option`:仅从指定 skill 的 observe inbox 信号生成草稿（仅 from-traces 模式用）。
@@ -689,7 +689,7 @@ omk studio --port 8080 --no-open
 | `construct` | 否 | 测的是什么构念 |
 | `provenance` | 否 | 用例来源（`omk sample` 自动打） |
 | `mocks` | 否 | 工具调用 mock 返回（sandbox 评测） |
-| `environment` | 否 | 评测环境前置「已就绪」声明 |
+| `environment` | 否 | 题设环境声明（仅注入 prompt，不物化） |
 | `tripwire` | 否 | 标记为「故意诱错」样本，failed 时 diagnostic 不建议改 skill |
 
 完整 schema 见 [docs/specs/sample-design-spec.md](https://github.com/lizhiyao/oh-my-knowledge/blob/main/docs/specs/sample-design-spec.md)。

package/dist/authoring/generator.d.ts

@@ -1,6 +1,7 @@
-import type { Sample, ExecutorFn } from '../types/index.js';
+import type { ExecutorFn, Sample } from '../types/index.js';
 import type { ObservationInboxItem } from '../types/observability.js';
-interface GenerateSamplesOptions {
+export declare function sampleGenerationUsesMocks(executorName: string | undefined, noMock?: boolean): boolean;
+export interface GenerateSamplesOptions {
     skillContent: string;
     count?: number;
     model: string;
@@ -12,6 +13,8 @@ interface GenerateSamplesOptions {
     focus?: string;
     /** 不生成 mocks/mocksStrict，eval 时真实执行所有工具调用。 */
     noMock?: boolean;
+    /** Injectable executor for tests. Defaults to createExecutor(executorName). */
+    executor?: ExecutorFn;
 }
 /**
  * 拼出送给 LLM 的 user prompt。抽出来便于单测验证 focus 是否真的注入了。
@@ -26,7 +29,7 @@ export declare function buildSamplesPrompt({ skillContent, count, focus, noMock
     focus?: string;
     noMock?: boolean;
 }): string;
-export declare function generateSamples({ skillContent, count, model, executorName, focus, noMock }: GenerateSamplesOptions): Promise<{
+export declare function generateSamples({ skillContent, count, model, executorName, focus, noMock, executor: injectedExecutor, }: GenerateSamplesOptions): Promise<{
     samples: Sample[];
     costUSD: number;
 }>;
@@ -62,12 +65,16 @@ export declare function stratifyTraceSignals(items: TraceSignalItem[]): Stratifi
  * production failures. The trace text feeds the *generator* only — never the
  * judge prompt — so judge-prompt isolation is unaffected.
  */
-export declare function buildSamplesFromTracesPrompt(items: TraceSignalItem[], count?: number): string;
+export declare function buildSamplesFromTracesPrompt(items: TraceSignalItem[], count?: number, options?: {
+    noMock?: boolean;
+}): string;
 export interface GenerateSamplesFromTracesOptions {
     items: TraceSignalItem[];
     count?: number;
     model: string;
     executorName?: string;
+    /** 不生成 mocks。目标 executor 不支持时会自动启用。 */
+    noMock?: boolean;
     /** Injectable executor (tests). Defaults to createExecutor(executorName). */
     executor?: ExecutorFn;
 }
@@ -77,12 +84,15 @@ export interface GenerateSamplesFromTracesOptions {
  * stamps `provenance: 'production-trace'`. Output is meant to land in a review draft,
  * not the live dataset (the CLI enforces that).
  */
-export declare function generateSamplesFromTraces({ items, count, model, executorName, executor: injectedExecutor, }: GenerateSamplesFromTracesOptions): Promise<{
+export declare function generateSamplesFromTraces({ items, count, model, executorName, noMock, executor: injectedExecutor, }: GenerateSamplesFromTracesOptions): Promise<{
     samples: Sample[];
     costUSD: number;
 }>;
 export declare function sanitizeGeneratedSamples(samples: Sample[], opts?: {
     skillContent?: string;
+    mockless?: boolean;
+    migrateMocklessEnvironment?: boolean;
+    preserveMocklessEnvironment?: boolean;
 }): {
     stripped: string[];
 };

package/dist/authoring/generator.js

@@ -1,5 +1,7 @@
 import { createExecutor } from '../executors/index.js';
+import { executorSupportsSampleMocks } from '../executors/capabilities.js';
 import { DEFAULT_GATE_THRESHOLD } from '../eval-core/verdict.js';
+import { sampleMockReferenceKeys } from '../shared/sample-contract.js';
 const SYSTEM_PROMPT = `你是一个评测用例生成器。你的任务是根据用户提供的 skill（系统提示词）内容，生成高质量的评测用例。
 
 样本结构决策（必须先做）：先扫一遍 skill 内容判断它属于哪一类，按对应配比和数量生成。
@@ -195,14 +197,14 @@ const SYSTEM_PROMPT = `你是一个评测用例生成器。你的任务是根据
         tool_input_not_contains 或 tools_not_called。
   - { "type": "regex", "pattern": "...", "weight": 1 }
         ↑ 同 contains 限制:只用在固定格式字面量(如 SHA / UUID / 路径模板)。
-- environment: 可选,对象。**评测环境的"已就绪"声明**,LLM 看到后跳过环境探测直接进工作流。
+- environment: 可选,对象。**仅作 prompt 上下文的题设环境声明**,不会修改 PATH、创建文件或物化 fixture。
   字段:
     - cli_available: string[],已在 PATH 上的 CLI(如 ["node", "git", "code-host"])
     - files_available: string[],已存在的文件/脚本(如 ["~/.req-tool-api.json", "$SKILL_DIR/scripts/x.js"])
     - notes: string,自由文本兜底(如"DevAPI 凭证有效,工号 testuser001")
   原则:
-    凡是 skill 跑起来需要的环境(凭证文件 / 业务 CLI / 自带脚本 / API token 等),
-    都写到这里,而不是在 mock 里 mock 它们的探测命令。这让 mock 只关注业务调用本身。
+    只有用例明确把某项环境能力作为题设前提时才写。需要读取真实内容的文件不能放在
+    files_available 里冒充 fixture:应放进 sample.cwd 下的真实 fixture,或由 mock 返回内容。
 - mocksStrict: **必填且必须设为 true**(只要 sample 配了 mocks)。
   原因:mocksStrict=false 时,LLM 调到没匹配 mock 的命令会**透传到真 shell**,
   既可能真调外部接口产生副作用,也可能因二进制不存在(如 mcporter)报噪声错误污染评测信号。
@@ -223,8 +225,8 @@ const SYSTEM_PROMPT = `你是一个评测用例生成器。你的任务是根据
            - 对(✅):写一条宽 mock:\`{tool:"Bash", match:{command_glob:"ls *"},
              return:{stdout:"<模拟目录列表>", exit:0}}\` — \`command_glob\` 用 \`*\` 兜底各种
              ls 参数变体(\`ls\` / \`ls -la\` / \`ls -d\` / \`ls /xx\` 全命中)
-       (c) 单纯"已就绪"声明(凭证文件 / 业务 CLI 是否安装)还是走 \`environment\` 字段,
-           不需要 LLM 真调命令检查 — environment 字段就是告诉 LLM "这些不用检查"。
+       (c) 题设明确声明可用的凭证 / CLI 可写进 \`environment\`,让 LLM 不做可用性探测。
+           该字段只进 prompt,不会真的安装 CLI、创建凭证文件或改变 runtime。
        (d) **intent-level mock(文件搜索/读取类操作)** — LLM 搜代码时会自由选择 Bash grep、
            Grep 工具、Glob+Read 组合、甚至 Agent 子代理,逐个枚举工具写 mock 不可持续。
            正确做法:用 \`tool: "*"\` + \`input_contains: "关键词"\` 按意图匹配:
@@ -367,6 +369,37 @@ const SYSTEM_PROMPT = `你是一个评测用例生成器。你的任务是根据
 - 字符串字段（prompt / rubric / capability 等）内部如需引号，**必须用全角「」**而不是半角 \`""\`，避免漏转义破坏 JSON 解析
 - 例：错 → \`"prompt": "查询"Daily"标签..."\`（内部 \`"\` 未转义，JSON 解析失败）
        对 → \`"prompt": "查询「Daily」标签..."\`（全角引号，无转义压力）`;
+const MOCKLESS_SYSTEM_OVERRIDE = `
+
+## 目标执行器能力约束（最高优先级）
+
+目标执行器不支持可靠的工具调用拦截。本节覆盖上文所有要求 mocks、mock_hit、
+files_available 或正向工具调用断言的规则：
+
+- 不要输出 mocks、mocksStrict、environment 字段。
+- 不要输出 mock_hit、tools_called、tools_count_min、tool_input_contains、tool_output_contains。
+- 可以保留 tools_not_called、tools_count_max、tool_input_not_contains 这类负向安全约束。
+- 把必要的输入事实直接写进 context，把可判分结果写进 rubric 或输出内容断言。
+- 不要假装某个文件、CLI 或工具调用已经由 fixture 物化。`;
+function generationSystemPrompt(mockless) {
+    return mockless ? `${SYSTEM_PROMPT}${MOCKLESS_SYSTEM_OVERRIDE}` : SYSTEM_PROMPT;
+}
+const warnedAutoMocklessExecutors = new Set();
+function warnAutoMockless(executorName, noMock) {
+    if (noMock
+        || !executorName
+        || executorSupportsSampleMocks(executorName)
+        || warnedAutoMocklessExecutors.has(executorName))
+        return;
+    warnedAutoMocklessExecutors.add(executorName);
+    process.stderr.write(`[omk sample] 执行器「${executorName}」不支持工具调用拦截，`
+        + '已自动切换为无 mocks 用例。\n');
+}
+export function sampleGenerationUsesMocks(executorName, noMock = false) {
+    if (noMock)
+        return false;
+    return executorName ? executorSupportsSampleMocks(executorName) : true;
+}
 /**
  * 拼出送给 LLM 的 user prompt。抽出来便于单测验证 focus 是否真的注入了。
  *
@@ -390,9 +423,12 @@ ${skillContent}
 
 ${countLine}直接输出 JSON 数组。${focusBlock}${noMockBlock}`;
 }
-export async function generateSamples({ skillContent, count, model, executorName, focus, noMock }) {
-    const executor = createExecutor(executorName);
-    const prompt = buildSamplesPrompt({ skillContent, count, focus, noMock });
+export async function generateSamples({ skillContent, count, model, executorName, focus, noMock, executor: injectedExecutor, }) {
+    const executor = injectedExecutor ?? createExecutor(executorName);
+    const mockless = !sampleGenerationUsesMocks(executorName, noMock);
+    warnAutoMockless(executorName, noMock);
+    const prompt = buildSamplesPrompt({ skillContent, count, focus, noMock: mockless });
+    const system = generationSystemPrompt(mockless);
     // 生成场景比单次 eval 调用更重(LLM 要思考结构 + 输出大段 JSON),
     // 默认 120s 对长 skill + count >= 8 经常不够,这里用 5 分钟兜底。
     // lean=true 关掉 agent 工具循环 / skill 发现 — 生成只需要纯文本,不需要 Bash / Read 等工具。
@@ -405,7 +441,7 @@ export async function generateSamples({ skillContent, count, model, executorName
         const attemptPrompt = attempt === 1
             ? prompt
             : `${prompt}\n\n上一次输出解析失败:${lastErr}\n请严格按 JSON 规范输出(字符串内部用「」全角引号),只输出数组,不要包含其他文字。`;
-        const result = await executor({ model, system: SYSTEM_PROMPT, prompt: attemptPrompt, timeoutMs: 300_000, lean: true });
+        const result = await executor({ model, system, prompt: attemptPrompt, timeoutMs: 300_000, lean: true });
         totalCost += result.costUSD || 0;
         if (!result.ok) {
             lastErr = result.error || 'unknown error';
@@ -435,12 +471,15 @@ export async function generateSamples({ skillContent, count, model, executorName
             continue;
         }
         // 通过校验,跳出循环继续后续 sanitize
-        return await finalizeSamples(samples, totalCost, skillContent);
+        return await finalizeSamples(samples, totalCost, {
+            skillContent,
+            mockless,
+        });
     }
     // 不可达 (循环里所有出口都 throw 或 return),保留是为了 TS 类型推断
     throw new Error('unreachable');
 }
-async function finalizeSamples(samples, costUSD, skillContent) {
+async function finalizeSamples(samples, costUSD, options) {
     // Validate required fields + sanitize metadata enums *at generator boundary*
     // (see sanitizeGeneratedSamples). skillContent is passed so the function can
     // strip "脑补"-style fact assertions whose tool name has no literal mention
@@ -448,7 +487,10 @@ async function finalizeSamples(samples, costUSD, skillContent) {
     // the data-security-review v1-v5 regen series (generator kept producing
     // tool_input_contains "WebFetch:语雀URL" even after 7 prompt iterations,
     // because LLM's "URL → fetch" training prior overrides instructional text).
-    const { stripped } = sanitizeGeneratedSamples(samples, { skillContent });
+    const { stripped } = sanitizeGeneratedSamples(samples, {
+        ...options,
+        migrateMocklessEnvironment: true,
+    });
     if (stripped.length > 0) {
         process.stderr.write(`[omk sample] LLM-output 含 ${stripped.length} 个非法元数据/断言字段，已剥离避免污染：\n  - ${stripped.join('\n  - ')}\n`);
     }
@@ -492,7 +534,7 @@ export function stratifyTraceSignals(items) {
  * production failures. The trace text feeds the *generator* only — never the
  * judge prompt — so judge-prompt isolation is unaffected.
  */
-export function buildSamplesFromTracesPrompt(items, count) {
+export function buildSamplesFromTracesPrompt(items, count, options = {}) {
     // 先按频次分层（合并重复 + 算占比 + 降序），让模型按「占比」分配配额，而非每信号一刀切。
     const stratified = stratifyTraceSignals(items);
     const sections = stratified.map((it, i) => {
@@ -515,13 +557,16 @@ export function buildSamplesFromTracesPrompt(items, count) {
     const countLine = typeof count === 'number'
         ? `共生成约 ${count} 条评测用例，按各信号的「占比」分配配额（高频多、低频少），覆盖整体失败分布。`
         : '按各信号「占比」分配：高频信号多生成、低频少生成，覆盖整体失败分布。';
+    const mocklessBlock = options.noMock
+        ? '\n\n目标执行器不支持工具调用拦截：不要生成 mocks、mocksStrict、environment 或正向工具调用断言；把 trace 证据写入 context 和 rubric。'
+        : '';
     return `${TRACE_GEN_INSTRUCTIONS}
 
 ## 观测到的失败信号（共 ${stratified.length} 个，已按出现频次降序）
 
 ${sections}
 
-${countLine}直接输出 JSON 数组。`;
+${countLine}直接输出 JSON 数组。${mocklessBlock}`;
 }
 /** Build a sanitize context string from trace evidence so finalizeSamples keeps
  *  tool-name assertions that reference tools actually seen in the traces (an empty
@@ -539,14 +584,17 @@ function traceSanitizeContext(items) {
  * stamps `provenance: 'production-trace'`. Output is meant to land in a review draft,
  * not the live dataset (the CLI enforces that).
  */
-export async function generateSamplesFromTraces({ items, count, model, executorName, executor: injectedExecutor, }) {
+export async function generateSamplesFromTraces({ items, count, model, executorName, noMock, executor: injectedExecutor, }) {
     if (items.length === 0)
         return { samples: [], costUSD: 0 };
     if (!injectedExecutor && !executorName) {
         throw new Error('executorName is required when no executor is injected');
     }
     const executor = injectedExecutor ?? createExecutor(executorName);
-    const prompt = buildSamplesFromTracesPrompt(items, count);
+    const mockless = !sampleGenerationUsesMocks(executorName, noMock);
+    warnAutoMockless(executorName, noMock);
+    const prompt = buildSamplesFromTracesPrompt(items, count, { noMock: mockless });
+    const system = generationSystemPrompt(mockless);
     const sanitizeContext = traceSanitizeContext(items);
     const PROVENANCE = 'production-trace';
     const MAX_ATTEMPTS = 3;
@@ -556,7 +604,7 @@ export async function generateSamplesFromTraces({ items, count, model, executorN
         const attemptPrompt = attempt === 1
             ? prompt
             : `${prompt}\n\n上一次输出解析失败:${lastErr}\n请严格按 JSON 规范输出(字符串内部用「」全角引号),只输出数组,不要包含其他文字。`;
-        const result = await executor({ model, system: SYSTEM_PROMPT, prompt: attemptPrompt, timeoutMs: 300_000, lean: true });
+        const result = await executor({ model, system, prompt: attemptPrompt, timeoutMs: 300_000, lean: true });
         totalCost += result.costUSD || 0;
         if (!result.ok) {
             lastErr = result.error || 'unknown error';
@@ -593,7 +641,10 @@ export async function generateSamplesFromTraces({ items, count, model, executorN
         // Stamp provenance before sanitize so it survives (it's a valid enum value).
         for (const s of samples)
             s.provenance = PROVENANCE;
-        return await finalizeSamples(samples, totalCost, sanitizeContext);
+        return await finalizeSamples(samples, totalCost, {
+            skillContent: sanitizeContext,
+            mockless,
+        });
     }
     throw new Error('unreachable');
 }
@@ -656,6 +707,78 @@ const TEXT_VALUE_TYPES = new Set([
 const TOOL_POSITIVE_TYPES = new Set([
     'tool_input_contains', 'tool_output_contains', 'mock_hit',
 ]);
+const MOCKLESS_POSITIVE_TOOL_TYPES = new Set([
+    'mock_hit',
+    'tools_called',
+    'tools_count_min',
+    'tool_input_contains',
+    'tool_output_contains',
+]);
+function stripMocklessPositiveToolAssertions(assertions, label, stripped) {
+    const kept = [];
+    for (const [index, assertion] of assertions.entries()) {
+        const assertionLabel = `${label}[${index}]`;
+        if (assertion?.type === 'assert-set' && Array.isArray(assertion.children)) {
+            const children = stripMocklessPositiveToolAssertions(assertion.children, `${assertionLabel}.children`, stripped);
+            if (children.length === 0) {
+                stripped.push(`${assertionLabel}.assert-set（无 mock 模式下没有可执行子断言）`);
+                continue;
+            }
+            assertion.children = children;
+            kept.push(assertion);
+            continue;
+        }
+        if (MOCKLESS_POSITIVE_TOOL_TYPES.has(assertion?.type)) {
+            stripped.push(`${assertionLabel}.${assertion.type}（目标执行器不支持 mocks，正向工具证据不可复现）`);
+            continue;
+        }
+        kept.push(assertion);
+    }
+    return kept;
+}
+function stripInvalidMockHitAssertions(assertions, mockKeys, label, stripped) {
+    const kept = [];
+    for (const [index, assertion] of assertions.entries()) {
+        const assertionLabel = `${label}[${index}]`;
+        if (assertion?.type === 'assert-set' && Array.isArray(assertion.children)) {
+            const children = stripInvalidMockHitAssertions(assertion.children, mockKeys, `${assertionLabel}.children`, stripped);
+            if (children.length === 0) {
+                stripped.push(`${assertionLabel}.assert-set（没有可执行子断言）`);
+                continue;
+            }
+            assertion.children = children;
+            kept.push(assertion);
+            continue;
+        }
+        if (assertion?.type === 'mock_hit'
+            && typeof assertion.value === 'string'
+            && !mockKeys.has(assertion.value)) {
+            stripped.push(`${assertionLabel}.mock_hit（未引用实际 mock：${assertion.value}）`);
+            continue;
+        }
+        kept.push(assertion);
+    }
+    return kept;
+}
+function environmentAsPromptContext(environment) {
+    if (!environment || typeof environment !== 'object' || Array.isArray(environment)) {
+        return null;
+    }
+    const env = environment;
+    const lines = ['题设环境声明（仅作上下文，不会在 cwd 物化）：'];
+    if (Array.isArray(env.cli_available)
+        && env.cli_available.every((item) => typeof item === 'string' && item.length > 0)) {
+        lines.push(`- 可用 CLI：${env.cli_available.join('、')}`);
+    }
+    if (Array.isArray(env.files_available)
+        && env.files_available.every((item) => typeof item === 'string' && item.length > 0)) {
+        lines.push(`- 题设引用路径（未物化）：${env.files_available.join('、')}`);
+    }
+    if (typeof env.notes === 'string' && env.notes.trim()) {
+        lines.push(`- 说明：${env.notes.trim()}`);
+    }
+    return lines.length > 1 ? lines.join('\n') : null;
+}
 function isAsciiTokenLike(v) {
     if (typeof v !== 'string')
         return false;
@@ -719,6 +842,40 @@ export function sanitizeGeneratedSamples(samples, opts = {}) {
             stripped.push(`samples[${i}].tripwire (${typeof s.tripwire})`);
             delete s.tripwire;
         }
+        if (opts.mockless) {
+            if (Array.isArray(s.mocks) && s.mocks.length > 0) {
+                stripped.push(`samples[${i}].mocks（目标执行器不支持工具调用拦截）`);
+            }
+            delete s.mocks;
+            if (s.mocksStrict !== undefined) {
+                stripped.push(`samples[${i}].mocksStrict（无 mocks）`);
+            }
+            delete s.mocksStrict;
+            if (s.environment !== undefined && !opts.preserveMocklessEnvironment) {
+                const environmentContext = opts.migrateMocklessEnvironment
+                    ? environmentAsPromptContext(s.environment)
+                    : null;
+                if (environmentContext) {
+                    const existingContext = typeof s.context === 'string' ? s.context.trim() : '';
+                    s.context = existingContext
+                        ? `${existingContext}\n\n${environmentContext}`
+                        : environmentContext;
+                    stripped.push(`samples[${i}].environment（已迁移到题设 context，未物化）`);
+                }
+                else {
+                    stripped.push(`samples[${i}].environment（未物化的环境声明不能充当 fixture）`);
+                }
+            }
+            if (!opts.preserveMocklessEnvironment) {
+                delete s.environment;
+            }
+            if (Array.isArray(s.assertions)) {
+                s.assertions = stripMocklessPositiveToolAssertions(s.assertions, `samples[${i}].assertions`, stripped);
+                if (s.assertions.length === 0) {
+                    delete s.assertions;
+                }
+            }
+        }
         // assertions 校验:loader 会拒掉两类无效断言 — 在 generator boundary 提前 strip,
         // 避免落盘的 sample 跑不动:
         //   1. tools_called / tools_not_called 的 values 必须非空
@@ -843,6 +1000,12 @@ export function sanitizeGeneratedSamples(samples, opts = {}) {
                     delete s.mocks;
             }
         }
+        if (Array.isArray(s.assertions)) {
+            s.assertions = stripInvalidMockHitAssertions(s.assertions, sampleMockReferenceKeys(s.mocks), `samples[${i}].assertions`, stripped);
+            if (s.assertions.length === 0) {
+                delete s.assertions;
+            }
+        }
         // mocksStrict 兜底:有 mocks 时强制 true。
         // SYSTEM_PROMPT 已要求 LLM 必填,但偶尔 LLM 漏填 — 在 generator boundary 修掉,
         // 避免运行时 mock 未命中透传到真 shell(报 mcporter not found 等噪声错误)。

package/dist/authoring/sample-fixer.d.ts

@@ -34,6 +34,8 @@ export interface FixSamplesOptions {
     }>;
     model: string;
     maxAttemptsPerSample?: number;
+    /** Target executor cannot intercept tool calls; strip mocks and dependent positive evidence. */
+    mockless?: boolean;
 }
 export interface FixSamplesResult {
     samples: Record<string, unknown>[];

package/dist/authoring/sample-fixer.js

@@ -20,8 +20,17 @@ const FIX_SYSTEM_PROMPT = `你是一个评测用例修复专家。根据诊断
 输出一个 **JSON 数组**,包含所有待修复 sample（修改过的和原样保留的都要包含）。
 第一字符 \`[\`，最后 \`]\`，不要用 \`\`\`json\`\`\` 围栏，不要寒暄。
 如果判断某条是 LLM 行为问题不需要改,也原样放进数组。`;
-function sanitizeFixedSamples(samples, skillContent) {
-    sanitizeGeneratedSamples(samples, { skillContent });
+const MOCKLESS_FIX_OVERRIDE = `
+
+目标执行器不支持工具调用拦截。不得新增或保留 mocks、mocksStrict、
+mock_hit、tools_called、tools_count_min、tool_input_contains、tool_output_contains。
+environment 仅表示题设上下文，不得当作已物化 fixture；负向安全断言可以保留。`;
+function sanitizeFixedSamples(samples, skillContent, mockless) {
+    sanitizeGeneratedSamples(samples, {
+        skillContent,
+        mockless,
+        preserveMocklessEnvironment: true,
+    });
     return samples;
 }
 const FIXABLE_SAMPLE_FIELDS = new Set([
@@ -97,7 +106,7 @@ function parseFixedSamples(text) {
     return null;
 }
 export async function fixSamples(options) {
-    const { skillContent, samples, report, treatmentKey, executor, model, maxAttemptsPerSample = 2 } = options;
+    const { skillContent, samples, report, treatmentKey, executor, model, maxAttemptsPerSample = 2, mockless = false, } = options;
     const sampleMap = new Map(samples.map((s) => [s.sample_id, s]));
     // Collect all fixable samples into one batch
     const fixContexts = [];
@@ -185,7 +194,12 @@ ${sampleSections}
     let incurredCostUSD = 0;
     let incurredCostReported = false;
     try {
-        const result = await executor({ model, system: FIX_SYSTEM_PROMPT, prompt, timeoutMs: 300_000 });
+        const result = await executor({
+            model,
+            system: mockless ? `${FIX_SYSTEM_PROMPT}${MOCKLESS_FIX_OVERRIDE}` : FIX_SYSTEM_PROMPT,
+            prompt,
+            timeoutMs: 300_000,
+        });
         incurredCostUSD = result.costUSD;
         incurredCostReported = result.costReported !== false;
         if (!result.ok) {
@@ -226,7 +240,7 @@ ${sampleSections}
                 })),
             };
         }
-        const sanitizedFixedArr = sanitizeFixedSamples(fixedArr, skillContent);
+        const sanitizedFixedArr = sanitizeFixedSamples(fixedArr, skillContent, mockless);
         const outOfScope = sanitizedFixedArr.find((fixed) => {
             const sid = fixed.sample_id;
             const original = sampleMap.get(sid);

package/dist/cli/commands/sample.js

@@ -274,7 +274,7 @@ async function runSampleFix(args, flags, lang) {
         throw new CliExit(1);
     }
     process.stderr.write(lang === 'zh' ? `🔧 发现 ${sampleDesignCount} 条 sample_design 失败，开始修复...\n` : `🔧 Found ${sampleDesignCount} sample_design failure(s), fixing...\n`);
-    const { createExecutor } = await import('../../executors/index.js');
+    const { createExecutor, executorSupportsSampleMocks, } = await import('../../executors/index.js');
     const exec = createExecutor(executorName);
     const executorFn = async (opts) => {
         const result = await exec({
@@ -298,6 +298,7 @@ async function runSampleFix(args, flags, lang) {
         treatmentKey: treatmentName,
         executor: executorFn,
         model,
+        mockless: !executorSupportsSampleMocks(executorName),
     });
     let writtenFiles = [];
     if (result.fixedCount > 0) {
@@ -363,7 +364,13 @@ export async function runSampleFromTraces(flags, lang) {
         ? `🔭 发现 ${items.length} 个${flags.skill ? ` ${flags.skill} 的` : ''}失败信号，正在生成评测用例草稿...\n`
         : `🔭 Found ${items.length}${flags.skill ? ` ${flags.skill}` : ''} failure signal(s); generating regression-sample drafts...\n`);
     try {
-        const { samples, costUSD } = await generateSamplesFromTraces({ items, count, model, executorName });
+        const { samples, costUSD } = await generateSamplesFromTraces({
+            items,
+            count,
+            model,
+            executorName,
+            noMock: flags['no-mock'],
+        });
         const cost = costUSD > 0 ? ` $${costUSD.toFixed(4)}` : '';
         if (samples.length === 0) {
             // The model conservatively skipped every signal (noise / unreproducible). That's a
@@ -663,8 +670,8 @@ export default class Sample extends BaseCommand {
         }),
         'no-mock': Flags.boolean({
             description: bilingual({
-                zh: '不生成 mocks，eval 时所有工具调用真实执行。',
-                en: 'Skip mock generation; all tool calls execute for real during eval.',
+                zh: '不生成 mocks。执行器不支持工具拦截时会自动启用，避免产生必然失败的 mock_hit。',
+                en: 'Skip mocks. Automatically enabled when the executor cannot intercept tools, preventing impossible mock_hit assertions.',
             }),
             default: false,
         }),

package/dist/eval-core/dependency-checker.js

@@ -25,7 +25,14 @@ const SYSTEM_ENV_VARS = new Set([
     'TMPDIR', 'EDITOR', 'VISUAL', 'HOSTNAME', 'LOGNAME', 'DISPLAY',
     'XDG_CONFIG_HOME', 'XDG_DATA_HOME', 'XDG_CACHE_HOME',
     'NODE_ENV', 'NODE_PATH', 'NODE_OPTIONS', 'NPM_CONFIG_PREFIX',
+    // Agent Skills resolves this placeholder to the active skill directory.
+    // It is not a user-provided environment prerequisite.
+    'SKILL_ROOT',
 ]);
+// A shell probe that explicitly tolerates failure is discovering the target
+// project shape. Candidate paths on that line are not hard dependencies of the
+// skill itself.
+const OPTIONAL_PROBE_LINE_REGEX = /(?:2>\s*\/dev\/null|\|\|\s*(?:true|:)(?:\s|$))/;
 function extractFromText(text) {
     const tools = new Set();
     const files = new Set();
@@ -42,25 +49,30 @@ function extractFromText(text) {
             tools.add(cmd);
         }
     }
-    // File paths
-    for (const match of text.matchAll(FILE_PATH_REGEX)) {
-        const path = match[1];
-        // Skip paths that look like URLs, package names, or version strings
-        if (path.startsWith('http') || path.startsWith('node_modules') || /^\d/.test(path))
+    // File paths. Keep line context so optional discovery commands do not turn
+    // every candidate project entry point into a fatal preflight requirement.
+    for (const line of text.split(/\r?\n/)) {
+        if (OPTIONAL_PROBE_LINE_REGEX.test(line))
             continue;
-        // Skip very short paths that are likely not real files
-        if (path.length < 5)
-            continue;
-        // Skip extension-mention patterns(`.d.ts` / `.tsx` 这种以点开头的"扩展名讨论"
-        // 不是真路径,SKILL.md 里"查看 .d.ts 文件"会被误识别)
-        if (path.startsWith('.'))
-            continue;
-        // Skip bare filenames without a directory segment(`index.ts` / `package.json`
-        // 这种通用文件名几乎都是示例性提及,真依赖会带路径段。要声明 bare 文件
-        // 走显式 requires)
-        if (!path.includes('/'))
-            continue;
-        files.add(path);
+        for (const match of line.matchAll(FILE_PATH_REGEX)) {
+            const path = match[1];
+            // Skip paths that look like URLs, package names, or version strings
+            if (path.startsWith('http') || path.startsWith('node_modules') || /^\d/.test(path))
+                continue;
+            // Skip very short paths that are likely not real files
+            if (path.length < 5)
+                continue;
+            // Skip extension-mention patterns(`.d.ts` / `.tsx` 这种以点开头的"扩展名讨论"
+            // 不是真路径,SKILL.md 里"查看 .d.ts 文件"会被误识别)
+            if (path.startsWith('.'))
+                continue;
+            // Skip bare filenames without a directory segment(`index.ts` / `package.json`
+            // 这种通用文件名几乎都是示例性提及,真依赖会带路径段。要声明 bare 文件
+            // 走显式 requires)
+            if (!path.includes('/'))
+                continue;
+            files.add(path);
+        }
     }
     // Environment variables
     for (const match of text.matchAll(ENV_VAR_REGEX)) {

package/dist/eval-core/evaluation-execution.js

@@ -258,8 +258,17 @@ export async function executeTasks({ tasks, executor, executorName, model, noJud
             }
         }
         let factCheck;
-        if (execResult.ok && execResult.output && task.cwd) {
-            factCheck = checkFacts(execResult.output, resolve(task.cwd));
+        if (execResult.ok && execResult.output) {
+            const sharedEvidence = {
+                ...(task._sample.context && { context: task._sample.context }),
+                ...(task._sample.environment?.files_available?.length && {
+                    declaredFiles: task._sample.environment.files_available,
+                }),
+                // Resolve exactly like the executor's cwd. samplesBaseDir is for bundle
+                // assets such as mocks, not for changing Sample.cwd path semantics.
+                ...(task._sample.cwd && { cwd: resolve(task._sample.cwd) }),
+            };
+            factCheck = checkFacts(execResult.output, sharedEvidence);
         }
         const sampleResults = ownRecordValue(results, task.sample_id)
             ?? setOwnRecordValue(results, task.sample_id, {});

package/dist/eval-core/fact-checker.d.ts

@@ -14,11 +14,24 @@ export interface FactCheckResult {
     totalCount: number;
     verifiedRate: number;
 }
+export interface FactCheckEvidence {
+    /** Shared sample fixture root. Arm-specific execution directories must not be used. */
+    cwd?: string;
+    /** Facts supplied to both arms in the sample context. */
+    context?: string;
+    /** Declarative fixture paths from sample.environment.files_available. */
+    declaredFiles?: string[];
+}
 /**
  * Extract file path claims from agent output text.
  */
 export declare function extractPathClaims(output: string): string[];
 /**
- * Check facts in agent output by verifying file paths exist in cwd.
+ * Check file-path facts against sample-level evidence shared by every arm.
+ *
+ * A string keeps the legacy direct-filesystem API for callers outside the
+ * evaluation pipeline. The pipeline passes structured evidence and deliberately
+ * excludes each artifact's execution cwd, because that directory is not a
+ * comparable source of truth across control and treatment arms.
  */
-export declare function checkFacts(output: string, cwd: string): FactCheckResult;
+export declare function checkFacts(output: string, evidence: string | FactCheckEvidence): FactCheckResult;