oh-my-knowledge 0.51.0 → 0.51.2

package/dist/observability/trace-attribution.js

@@ -1,4 +1,5 @@
 /** Skill attribution rules for trace records. */
+import { extractCodexExecCommands } from './codex-exec-command.js';
 export function extractMarkdownLogSkill(text) {
     const patterns = [
         /\b(?:prefer|use|call|invoke)\s+`?([a-zA-Z0-9][\w.-]*)`?\s+skill\b/i,
@@ -192,108 +193,6 @@ function extractInstalledSkillReadRef(text) {
     }
     return null;
 }
-function skipJsString(source, start) {
-    const quote = source[start];
-    let index = start + 1;
-    while (index < source.length) {
-        if (source[index] === '\\') {
-            index += 2;
-            continue;
-        }
-        if (source[index] === quote)
-            return index + 1;
-        index += 1;
-    }
-    return source.length;
-}
-function skipJsTrivia(source, start) {
-    let index = start;
-    while (index < source.length) {
-        if (/\s/.test(source[index])) {
-            index += 1;
-            continue;
-        }
-        if (source.startsWith('//', index)) {
-            const newline = source.indexOf('\n', index + 2);
-            return newline < 0 ? source.length : skipJsTrivia(source, newline + 1);
-        }
-        if (source.startsWith('/*', index)) {
-            const end = source.indexOf('*/', index + 2);
-            return end < 0 ? source.length : skipJsTrivia(source, end + 2);
-        }
-        break;
-    }
-    return index;
-}
-function extractExecCommandLiteral(source, callStart) {
-    let index = skipJsTrivia(source, callStart + 'tools.exec_command'.length);
-    if (source[index] !== '(')
-        return null;
-    index = skipJsTrivia(source, index + 1);
-    if (source[index] !== '{')
-        return null;
-    let depth = 1;
-    index += 1;
-    while (index < source.length && depth > 0) {
-        index = skipJsTrivia(source, index);
-        const char = source[index];
-        if (char === '"' || char === "'" || char === '`') {
-            index = skipJsString(source, index);
-            continue;
-        }
-        if (char === '{') {
-            depth += 1;
-            index += 1;
-            continue;
-        }
-        if (char === '}') {
-            depth -= 1;
-            index += 1;
-            continue;
-        }
-        if (depth === 1
-            && source.startsWith('cmd', index)
-            && !/[\w$]/.test(source[index - 1] ?? '')
-            && !/[\w$]/.test(source[index + 3] ?? '')) {
-            let valueStart = skipJsTrivia(source, index + 3);
-            if (source[valueStart] !== ':') {
-                index += 3;
-                continue;
-            }
-            valueStart = skipJsTrivia(source, valueStart + 1);
-            const quote = source[valueStart];
-            if (quote !== '"' && quote !== "'" && quote !== '`')
-                return null;
-            const end = skipJsString(source, valueStart);
-            return source.slice(valueStart + 1, Math.max(valueStart + 1, end - 1));
-        }
-        index += 1;
-    }
-    return null;
-}
-function extractExecCommandLiterals(source) {
-    const commands = [];
-    let index = 0;
-    while (index < source.length) {
-        index = skipJsTrivia(source, index);
-        const char = source[index];
-        if (char === '"' || char === "'" || char === '`') {
-            index = skipJsString(source, index);
-            continue;
-        }
-        if (source.startsWith('tools.exec_command', index)
-            && !/[\w$.]/.test(source[index - 1] ?? '')
-            && !/[\w$]/.test(source[index + 'tools.exec_command'.length] ?? '')) {
-            const command = extractExecCommandLiteral(source, index);
-            if (command)
-                commands.push(command);
-            index += 'tools.exec_command'.length;
-            continue;
-        }
-        index += 1;
-    }
-    return commands;
-}
 function splitShellCommandSegments(command) {
     const segments = [];
     let start = 0;
@@ -375,7 +274,7 @@ export function extractSkillReadFileRef(record) {
                 ? []
                 : part.name === 'Bash'
                     ? [rawCommand]
-                    : extractExecCommandLiterals(rawCommand);
+                    : extractCodexExecCommands(rawCommand);
             for (const command of commands) {
                 const skillRef = extractShellSkillReadRef(command);
                 if (skillRef)
@@ -408,7 +307,7 @@ export function extractSkillScriptCommandRef(record) {
                     : part.input?.code ?? part.input?.command;
                 if (typeof rawCommand === 'string') {
                     texts.push(...(part.name?.toLowerCase() === 'exec'
-                        ? extractExecCommandLiterals(rawCommand)
+                        ? extractCodexExecCommands(rawCommand)
                         : [rawCommand]));
                 }
             }
@@ -465,9 +364,14 @@ export function extractSkillReadFileRefFromEvent(event) {
         || sourceName === 'js'
         || event.tool.name === 'node_repl.js'
         || event.tool.name.toLowerCase() === 'js';
-    const commands = isOrchestrationWrapper
-        ? extractExecCommandLiterals(rawCommand)
-        : [rawCommand];
+    const normalizedCommands = Array.isArray(event.input.commands)
+        ? event.input.commands.filter((value) => typeof value === 'string')
+        : [];
+    const commands = normalizedCommands.length > 0
+        ? normalizedCommands
+        : isOrchestrationWrapper
+            ? extractCodexExecCommands(rawCommand)
+            : [rawCommand];
     for (const command of commands) {
         const skillRef = extractShellSkillReadRef(command);
         if (skillRef)

package/dist/server/skill-insights.js

@@ -250,7 +250,7 @@ function detectSkillDocGap(doctor, evalReport, observe, variantName) {
     const recs = [];
     if (depRule) {
         recs.push({
-            action: `在 sample.environment.files_available 加上文件路径,告诉 LLM "这些文件已就绪,无需探测"`,
+            action: '把已知路径写进 sample.context；只有纯题设前提才放 environment.files_available，且不要把它当作已物化 fixture',
             priority: severity,
             patch: {
                 target: 'sample-environment',

package/dist/shared/sample-contract.d.ts

@@ -1,3 +1,4 @@
 export declare function assertionContractValidationError(value: unknown, depth?: number, insideAssertSet?: boolean): string | undefined;
+export declare function sampleMockReferenceKeys(value: unknown): ReadonlySet<string>;
 export declare function dependencyRequirementsValidationError(value: unknown): string | undefined;
 export declare function sampleContractValidationError(value: unknown, expectedId?: string): string | undefined;

package/dist/shared/sample-contract.js

@@ -219,6 +219,38 @@ function mockValidationError(value) {
         return '"match.input" must be a JSON object when present';
     return undefined;
 }
+export function sampleMockReferenceKeys(value) {
+    const keys = new Set();
+    if (!Array.isArray(value))
+        return keys;
+    const countByTool = new Map();
+    for (const mock of value) {
+        if (!isRecord(mock) || !isNonEmptyString(mock.tool))
+            continue;
+        const ordinal = (countByTool.get(mock.tool) ?? 0) + 1;
+        countByTool.set(mock.tool, ordinal);
+        keys.add(`${mock.tool}:${ordinal}`);
+    }
+    return keys;
+}
+function mockHitReferenceValidationError(assertions, mockKeys) {
+    if (!Array.isArray(assertions))
+        return undefined;
+    for (const assertion of assertions) {
+        if (!isRecord(assertion))
+            continue;
+        if (assertion.type === 'mock_hit'
+            && typeof assertion.value === 'string'
+            && !mockKeys.has(assertion.value)) {
+            const available = mockKeys.size > 0 ? [...mockKeys].join(', ') : '(none)';
+            return `"mock_hit" references missing mock ${JSON.stringify(assertion.value)}; available mock keys: ${available}`;
+        }
+        const childError = mockHitReferenceValidationError(assertion.children, mockKeys);
+        if (childError)
+            return childError;
+    }
+    return undefined;
+}
 export function dependencyRequirementsValidationError(value) {
     if (!isRecord(value))
         return '"requires" must be an object';
@@ -276,6 +308,9 @@ export function sampleContractValidationError(value, expectedId) {
                 return `"mocks[${index}]": ${error}`;
         }
     }
+    const mockHitError = mockHitReferenceValidationError(value.assertions, sampleMockReferenceKeys(value.mocks));
+    if (mockHitError)
+        return mockHitError;
     if (value.mocksStrict !== undefined && typeof value.mocksStrict !== 'boolean') {
         return '"mocksStrict" must be boolean when present';
     }

package/dist/shared/tool-identity.d.ts

@@ -19,3 +19,11 @@ export interface ToolIdentityInput {
  * retaining source identity for audit and future protocol migrations.
  */
 export declare function normalizeToolIdentity(input: ToolIdentityInput): NormalizedToolIdentity;
+/**
+ * Match a source-neutral tool identity against a runtime-native tool name.
+ *
+ * Exact matching remains first for legacy/custom tools. Normalization then lets
+ * one mock identity work across adapters such as `Bash` ↔ `exec_command` and
+ * `Edit` ↔ `apply_patch`.
+ */
+export declare function toolIdentityMatches(expectedName: string, runtimeName: string): boolean;

package/dist/shared/tool-identity.js

@@ -62,6 +62,19 @@ export function normalizeToolIdentity(input) {
         ...(sourceName !== name ? { sourceName } : {}),
     };
 }
+/**
+ * Match a source-neutral tool identity against a runtime-native tool name.
+ *
+ * Exact matching remains first for legacy/custom tools. Normalization then lets
+ * one mock identity work across adapters such as `Bash` ↔ `exec_command` and
+ * `Edit` ↔ `apply_patch`.
+ */
+export function toolIdentityMatches(expectedName, runtimeName) {
+    if (expectedName === runtimeName)
+        return true;
+    return normalizeToolIdentity({ sourceName: expectedName }).name
+        === normalizeToolIdentity({ sourceName: runtimeName }).name;
+}
 function inferredMcpNamespace(sourceName) {
     const parts = sourceName.split('__').filter(Boolean);
     return parts[0] === 'mcp' && parts.length > 2

package/dist/types/eval.d.ts

@@ -35,12 +35,12 @@ export interface SampleCoverageTarget {
     /** 稳定引用：路径或 ID。reference/script 用 skill 根相对路径，workflow_node 用 workflowId.nodeId。 */
     ref: string;
 }
-/** Sample 评测环境前置:声明性"已就绪"清单,LLM 看到后跳过环境探测,直接进入工作流。
- *  类比 unit test 的 fixture / setup —— 评测是测 skill 工作流,不是测环境探测能力。 */
+/** Sample 题设环境声明。仅注入 prompt,不会修改 PATH、物化文件或改变 runtime。 */
 export interface SampleEnvironment {
-    /** 假定已在 PATH 上的 CLI,LLM 不再 which / find / type / command -v 探测。 */
+    /** 题设声明可用的 CLI,LLM 不再 which / find / type / command -v 探测。 */
     cli_available?: string[];
-    /** 假定存在的文件/脚本(支持 ~ / $SKILL_DIR / 绝对路径),LLM 不再 Glob / Read / test -f 探测。 */
+    /** 题设声明存在的文件/脚本(支持 ~ / $SKILL_DIR / 绝对路径),LLM 不再 Glob / Read / test -f 探测。
+     *  这是 prompt context,不会在 cwd 物化文件;需要真实读取时必须提供 sample.cwd 中的 fixture。 */
     files_available?: string[];
     /** 自由文本兜底,场景特殊说明(如"凭证已配""设备 SN xxx 已租"等)。 */
     notes?: string;
@@ -80,7 +80,8 @@ export type MockReturn = {
 } | string;
 /** 单条 Mock 规则。runtime 拦到匹配的 tool 调用即返回 mocked 结果,不放出去。 */
 export interface Mock {
-    /** 拦截的工具名,如 "Read" / "Bash" / "WebFetch" / "Edit" / "Write" / "Grep" / "Glob"。
+    /** source-neutral 工具身份,如 "Read" / "Bash" / "WebFetch" / "Edit" / "Write" / "Grep" / "Glob"。
+     *  executor adapter 会把 runtime-native 名称（如 exec_command / apply_patch）映射后匹配。
      *  特殊值 `"*"`:通配,匹配任何工具名(配合 match.input_contains 做 intent-level mock)。 */
     tool: string;
     /** 命中规则。所有字段 AND,字段未填即不限制。 */
@@ -136,7 +137,7 @@ export interface Sample {
      *  - true:未命中即 deny(防意外真调外部接口/CLI/MCP/写状态)。
      *  全 mock 评测场景建议 true,部分 mock 探索场景留 false。 */
     mocksStrict?: boolean;
-    /** 环境前置:声明性"已就绪"清单,LLM 跳过探测直接干活。详见 SampleEnvironment。 */
+    /** 题设环境声明,仅作 prompt 上下文。详见 SampleEnvironment。 */
     environment?: SampleEnvironment;
     [key: string]: unknown;
 }

package/dist/types/executor.d.ts

@@ -87,8 +87,9 @@ export interface ExecutorInput {
      *  - claude-sdk:转 in-process HookCallback 装到 SDK options.hooks.PreToolUse
      *  - claude-cli:物化为临时 settings.json + on-disk hook 脚本,跑完清理
      *  - script(自定义脚本):同样物化临时 settings,通过 env(OMK_MOCK_SETTINGS_FILE /
-     *    OMK_MOCK_MCP_CONFIG_FILE / OMK_MOCKS_FILE)暴露给脚本;脚本若包 Claude Code 兼容
-     *    CLI 可透传 --settings 复用同一 mock hook,否则忽略(不支持的 CLI 静默无 mock) */
+     *    OMK_MOCK_MCP_CONFIG_FILE / OMK_MOCKS_FILE)暴露给脚本;脚本负责消费该协议
+     *  - codex / codex-sdk / gemini / *-api:不支持,executor capability gate 会拒绝,
+     *    绝不静默忽略后把 mock_hit 记成模型失败 */
     mocks?: import('./eval.js').Mock[];
     /** 解析 mock.return_file 的相对路径锚点(默认 sample 文件所在目录)。 */
     mocksBaseDir?: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "oh-my-knowledge",
-  "version": "0.51.0",
+  "version": "0.51.2",
   "packageManager": "yarn@4.16.0",
   "description": "Evaluation framework for LLM knowledge inputs — prompts, RAG corpora, skills, agent workflows. Fix the model, vary the artifact. Built-in statistical rigor: bootstrap CI, Krippendorff α, length-debias, saturation curves.",
   "type": "module",
@@ -96,11 +96,11 @@
   "license": "MIT",
   "dependencies": {
     "@anthropic-ai/claude-agent-sdk": "^0.3.143",
-    "@anthropic-ai/sdk": "^0.112.3",
+    "@anthropic-ai/sdk": "^0.115.0",
     "@inquirer/prompts": "^8.4.3",
     "@modelcontextprotocol/sdk": "^1.29.0",
     "@oclif/core": "^4",
-    "@openai/codex-sdk": "0.144.6",
+    "@openai/codex-sdk": "0.145.0",
     "ajv": "^8.18.0",
     "chart.js": "^4.5.1",
     "es-module-lexer": "^2.0.0",
@@ -116,7 +116,7 @@
     "@types/node": "^25.5.0",
     "eslint": "^10.1.0",
     "husky": "^9.1.7",
-    "lint-staged": "17.1.0",
+    "lint-staged": "17.2.0",
     "npm-run-all2": "^9.0.1",
     "typescript": "^6.0.2",
     "typescript-eslint": "^8.58.0",