oh-my-knowledge 0.30.0 → 0.32.0

package/dist/eval-core/execution-strategy.js

@@ -0,0 +1,165 @@
+import { dirname, join } from 'node:path';
+import { existsSync, mkdirSync } from 'node:fs';
+import { homedir } from 'node:os';
+/**
+ * Isolated cwd for strict baseline. Empty dir under ~/.oh-my-knowledge/
+ * so baseline's Glob/Read tools can't walk into the user's eval workdir (which
+ * usually has skills/<name>/ symlinks for treatment variants — those symlinks
+ * leak the skill content into baseline via plain file-system access, even when
+ * Skill auto-discovery is blocked).
+ *
+ * Stable path (not per-task tmpdir) so cache keys remain consistent.
+ */
+function getIsolatedCwd() {
+    const dir = join(homedir(), '.oh-my-knowledge', 'isolated-cwd');
+    if (!existsSync(dir)) {
+        mkdirSync(dir, { recursive: true });
+    }
+    return dir;
+}
+function combineUserPrompt(prefix, prompt) {
+    if (!prefix)
+        return prompt;
+    return `${prefix}\n\n${prompt}`;
+}
+export function resolveArtifactExecutionStrategy(artifact) {
+    switch (artifact.kind) {
+        case 'baseline':
+            return 'baseline';
+        case 'prompt':
+            return 'user-prompt';
+        case 'agent':
+            return 'agent-session';
+        case 'workflow':
+            return 'workflow-session';
+        case 'skill':
+        default:
+            return 'system-prompt';
+    }
+}
+export function resolveExperimentType(artifact) {
+    if (artifact.kind === 'baseline' && !artifact.cwd)
+        return 'baseline';
+    if (artifact.kind === 'baseline' && artifact.cwd)
+        return 'runtime-context-only';
+    return 'artifact-injection';
+}
+export function buildVariantConfig(artifact) {
+    // experimentRole 是 variant 的 run-time 属性,必须由上游显式注入:
+    //   - CLI:`--control` / `--treatment` 在 evaluation-preparation.ts 按 spec.name 匹配填
+    //   - batch workflow:显式标 baseline=control / skill=treatment
+    //   - config:configVariantsToSpecs 从 EvalConfig.variants.role 带入
+    // 静默从 artifactKind 反推会违反 terminology-spec 三-4"experimentRole 是唯一来源",
+    // 因此缺失时直接 throw,强迫调用者修配置,不走默认值掩盖。
+    if (!artifact.experimentRole) {
+        throw new Error(`artifact "${artifact.name}" 缺少 experimentRole:应该由 CLI --control/--treatment、`
+            + `--config eval.yaml 或 batch workflow 显式注入。参见 docs/specs/terminology-spec.md 三-4。`);
+    }
+    return {
+        variant: artifact.name,
+        artifactKind: artifact.kind,
+        artifactSource: artifact.source,
+        executionStrategy: resolveArtifactExecutionStrategy(artifact),
+        experimentType: resolveExperimentType(artifact),
+        experimentRole: artifact.experimentRole,
+        hasArtifactContent: Boolean(artifact.content),
+        cwd: artifact.cwd || null,
+        locator: artifact.locator,
+        ref: artifact.ref,
+        // propagate skill-isolation declaration so report.meta.skillIsolation
+        // 能在 evaluation-reporting 阶段从 variantConfigs 提取 (avoid re-resolving artifacts).
+        ...(artifact.allowedSkills !== undefined && { allowedSkills: artifact.allowedSkills }),
+    };
+}
+function extractSkillDir(artifact) {
+    if (!artifact.locator)
+        return null;
+    return dirname(artifact.locator);
+}
+export function resolveExecutionStrategy(task, model, timeoutMs, verbose, effort, samplesBaseDir) {
+    const skillDir = extractSkillDir(task.artifact);
+    // strict-baseline cwd 沙箱:baseline + allowedSkills===[] + 用户没显式
+    // cwd 时,改用 isolated empty dir。否则 baseline 的 Glob/Read 会走进用户工作目录
+    // (含 skills/<name>/ symlink) 直接读到 skill 内容,绕过 SDK isolation。
+    // 用户显式给 baseline 设了 cwd 时不动(用户自己负责干净)。
+    const isStrictBaseline = task.artifact.kind === 'baseline'
+        && Array.isArray(task.artifact.allowedSkills)
+        && task.artifact.allowedSkills.length === 0;
+    const effectiveCwd = isStrictBaseline && !task.cwd ? getIsolatedCwd() : task.cwd;
+    const baseInput = {
+        model,
+        cwd: effectiveCwd,
+        skillDir,
+        timeoutMs,
+        verbose,
+        ...(effort && { effort }),
+        // pass skill-isolation declaration to executors. undefined keeps
+        // SDK default; [] = strict isolation (skills:[] + disallowedTools:['Skill']);
+        // [...] = whitelist (skills:[...] only).
+        ...(task.artifact.allowedSkills !== undefined && { allowedSkills: task.artifact.allowedSkills }),
+        // Sample.mocks 透传到 executor。executor(claude-sdk / claude-cli)
+        // 自决定怎么落地(in-process hook vs 临时 CLAUDE_CONFIG_DIR + on-disk hook)。
+        // mocksBaseDir 优先 samplesBaseDir(sample bundle 根,跟 ExecutorInput 类型注释一致),
+        // 让 mock.return_file 的相对路径相对 sample 文件所在目录解析;
+        // 没传时 fallback skillDir → effectiveCwd,兼容老调用。
+        ...(task._sample.mocks && task._sample.mocks.length > 0 && {
+            mocks: task._sample.mocks,
+            mocksBaseDir: samplesBaseDir || skillDir || effectiveCwd || undefined,
+            ...(task._sample.mocksStrict && { mocksStrict: true }),
+        }),
+    };
+    switch (task.artifact.kind) {
+        case 'baseline':
+            return {
+                strategy: resolveArtifactExecutionStrategy(task.artifact),
+                cacheSystem: '',
+                input: {
+                    ...baseInput,
+                    system: null,
+                    prompt: task.prompt,
+                },
+            };
+        case 'prompt':
+            return {
+                strategy: resolveArtifactExecutionStrategy(task.artifact),
+                cacheSystem: '',
+                input: {
+                    ...baseInput,
+                    system: null,
+                    prompt: combineUserPrompt(task.artifact.content, task.prompt),
+                },
+            };
+        case 'agent':
+            return {
+                strategy: resolveArtifactExecutionStrategy(task.artifact),
+                cacheSystem: task.artifact.content ?? '',
+                input: {
+                    ...baseInput,
+                    system: task.artifact.content,
+                    prompt: task.prompt,
+                },
+            };
+        case 'workflow':
+            return {
+                strategy: resolveArtifactExecutionStrategy(task.artifact),
+                cacheSystem: task.artifact.content ?? '',
+                input: {
+                    ...baseInput,
+                    system: task.artifact.content,
+                    prompt: task.prompt,
+                },
+            };
+        case 'skill':
+        default:
+            return {
+                strategy: resolveArtifactExecutionStrategy(task.artifact),
+                cacheSystem: task.artifact.content ?? '',
+                input: {
+                    ...baseInput,
+                    system: task.artifact.content,
+                    prompt: task.prompt,
+                },
+            };
+    }
+}
+//# sourceMappingURL=execution-strategy.js.map

package/dist/eval-core/execution-strategy.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"execution-strategy.js","sourceRoot":"","sources":["../../src/eval-core/execution-strategy.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAGlC;;;;;;;;GAQG;AACH,SAAS,cAAc;IACrB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,kBAAkB,EAAE,cAAc,CAAC,CAAC;IAChE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACrB,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAQD,SAAS,iBAAiB,CAAC,MAAqB,EAAE,MAAc;IAC9D,IAAI,CAAC,MAAM;QAAE,OAAO,MAAM,CAAC;IAC3B,OAAO,GAAG,MAAM,OAAO,MAAM,EAAE,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,gCAAgC,CAAC,QAAkB;IACjE,QAAQ,QAAQ,CAAC,IAAI,EAAE,CAAC;QACtB,KAAK,UAAU;YACb,OAAO,UAAU,CAAC;QACpB,KAAK,QAAQ;YACX,OAAO,aAAa,CAAC;QACvB,KAAK,OAAO;YACV,OAAO,eAAe,CAAC;QACzB,KAAK,UAAU;YACb,OAAO,kBAAkB,CAAC;QAC5B,KAAK,OAAO,CAAC;QACb;YACE,OAAO,eAAe,CAAC;IAC3B,CAAC;AACH,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,QAAkB;IACtD,IAAI,QAAQ,CAAC,IAAI,KAAK,UAAU,IAAI,CAAC,QAAQ,CAAC,GAAG;QAAE,OAAO,UAAU,CAAC;IACrE,IAAI,QAAQ,CAAC,IAAI,KAAK,UAAU,IAAI,QAAQ,CAAC,GAAG;QAAE,OAAO,sBAAsB,CAAC;IAChF,OAAO,oBAAoB,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,QAAkB;IACnD,oDAAoD;IACpD,kFAAkF;IAClF,4DAA4D;IAC5D,iEAAiE;IACjE,qEAAqE;IACrE,kCAAkC;IAClC,IAAI,CAAC,QAAQ,CAAC,cAAc,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CACb,aAAa,QAAQ,CAAC,IAAI,oDAAoD;cAC5E,iFAAiF,CACpF,CAAC;IACJ,CAAC;IACD,OAAO;QACL,OAAO,EAAE,QAAQ,CAAC,IAAI;QACtB,YAAY,EAAE,QAAQ,CAAC,IAAI;QAC3B,cAAc,EAAE,QAAQ,CAAC,MAAM;QAC/B,iBAAiB,EAAE,gCAAgC,CAAC,QAAQ,CAAC;QAC7D,cAAc,EAAE,qBAAqB,CAAC,QAAQ,CAAC;QAC/C,cAAc,EAAE,QAAQ,CAAC,cAAc;QACvC,kBAAkB,EAAE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;QAC7C,GAAG,EAAE,QAAQ,CAAC,GAAG,IAAI,IAAI;QACzB,OAAO,EAAE,QAAQ,CAAC,OAAO;QACzB,GAAG,EAAE,QAAQ,CAAC,GAAG;QACjB,sEAAsE;QACtE,gFAAgF;QAChF,GAAG,CAAC,QAAQ,CAAC,aAAa,KAAK,SAAS,IAAI,EAAE,aAAa,EAAE,QAAQ,CAAC,aAAa,EAAE,CAAC;KACvF,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,QAAkB;IACzC,IAAI,CAAC,QAAQ,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IACnC,OAAO,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,IAAU,EAAE,KAAa,EAAE,SAAkB,EAAE,OAAiB,EAAE,MAAoD,EAAE,cAAuB;IACtL,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAChD,+DAA+D;IAC/D,gEAAgE;IAChE,6DAA6D;IAC7D,uCAAuC;IACvC,MAAM,gBAAgB,GACpB,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,UAAU;WAC9B,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC;WAC1C,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,KAAK,CAAC,CAAC;IAC9C,MAAM,YAAY,GAAG,gBAAgB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;IACjF,MAAM,SAAS,GAAG;QAChB,KAAK;QACL,GAAG,EAAE,YAAY;QACjB,QAAQ;QACR,SAAS;QACT,OAAO;QACP,GAAG,CAAC,MAAM,IAAI,EAAE,MAAM,EAAE,CAAC;QACzB,iEAAiE;QACjE,8EAA8E;QAC9E,yCAAyC;QACzC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,aAAa,KAAK,SAAS,IAAI,EAAE,aAAa,EAAE,IAAI,CAAC,QAAQ,CAAC,aAAa,EAAE,CAAC;QAChG,8DAA8D;QAC9D,mEAAmE;QACnE,0EAA0E;QAC1E,8CAA8C;QAC9C,8CAA8C;QAC9C,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI;YACzD,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK;YACzB,YAAY,EAAE,cAAc,IAAI,QAAQ,IAAI,YAAY,IAAI,SAAS;YACrE,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;SACvD,CAAC;KACH,CAAC;IAEF,QAAQ,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC3B,KAAK,UAAU;YACb,OAAO;gBACL,QAAQ,EAAE,gCAAgC,CAAC,IAAI,CAAC,QAAQ,CAAC;gBACzD,WAAW,EAAE,EAAE;gBACf,KAAK,EAAE;oBACL,GAAG,SAAS;oBACZ,MAAM,EAAE,IAAI;oBACZ,MAAM,EAAE,IAAI,CAAC,MAAM;iBACpB;aACF,CAAC;QACJ,KAAK,QAAQ;YACX,OAAO;gBACL,QAAQ,EAAE,gCAAgC,CAAC,IAAI,CAAC,QAAQ,CAAC;gBACzD,WAAW,EAAE,EAAE;gBACf,KAAK,EAAE;oBACL,GAAG,SAAS;oBACZ,MAAM,EAAE,IAAI;oBACZ,MAAM,EAAE,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC;iBAC9D;aACF,CAAC;QACJ,KAAK,OAAO;YACV,OAAO;gBACL,QAAQ,EAAE,gCAAgC,CAAC,IAAI,CAAC,QAAQ,CAAC;gBACzD,WAAW,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,EAAE;gBACxC,KAAK,EAAE;oBACL,GAAG,SAAS;oBACZ,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO;oBAC7B,MAAM,EAAE,IAAI,CAAC,MAAM;iBACpB;aACF,CAAC;QACJ,KAAK,UAAU;YACb,OAAO;gBACL,QAAQ,EAAE,gCAAgC,CAAC,IAAI,CAAC,QAAQ,CAAC;gBACzD,WAAW,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,EAAE;gBACxC,KAAK,EAAE;oBACL,GAAG,SAAS;oBACZ,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO;oBAC7B,MAAM,EAAE,IAAI,CAAC,MAAM;iBACpB;aACF,CAAC;QACJ,KAAK,OAAO,CAAC;QACb;YACE,OAAO;gBACL,QAAQ,EAAE,gCAAgC,CAAC,IAAI,CAAC,QAAQ,CAAC;gBACzD,WAAW,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,EAAE;gBACxC,KAAK,EAAE;oBACL,GAAG,SAAS;oBACZ,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO;oBAC7B,MAAM,EAAE,IAAI,CAAC,MAAM;iBACpB;aACF,CAAC;IACN,CAAC;AACH,CAAC"}

package/dist/eval-core/fact-checker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fact-checker.d.ts","sourceRoot":"","sources":["../../src/eval-core/fact-checker.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,WAAW,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;CACtB;AAsBD;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAmB1D;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,eAAe,CAmBvE"}

package/dist/eval-core/fact-checker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fact-checker.js","sourceRoot":"","sources":["../../src/eval-core/fact-checker.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAgBpC,+DAA+D;AAC/D,MAAM,aAAa,GAAG;IACpB,iEAAiE;IACjE,qEAAqE;IACrE,gBAAgB;IAChB,8BAA8B;IAC9B,+CAA+C;IAC/C,oEAAoE;CACrE,CAAC;AAEF,kDAAkD;AAClD,MAAM,eAAe,GAAG;IACtB,iBAAiB;IACjB,kBAAkB;IAClB,iBAAiB;IACjB,SAAS;IACT,UAAU;IACV,cAAc;CACf,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAAc;IAC9C,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAEhC,KAAK,MAAM,OAAO,IAAI,aAAa,EAAE,CAAC;QACpC,mCAAmC;QACnC,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;QACtB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YAC7C,IAAI,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACpB,6BAA6B;YAC7B,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;YACxC,gCAAgC;YAChC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;YACnC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;gBAClE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAClB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC;AACpB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,MAAc,EAAE,GAAW;IACpD,MAAM,UAAU,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;IAE7C,MAAM,MAAM,GAAgB,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAClD,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QACpC,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;QACpC,OAAO;YACL,IAAI,EAAE,WAAW;YACjB,KAAK,EAAE,IAAI;YACX,QAAQ,EAAE,MAAM;YAChB,GAAG,CAAC,CAAC,MAAM,IAAI,EAAE,QAAQ,EAAE,GAAG,QAAQ,YAAY,EAAE,CAAC;SACtD,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;IAC9D,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;IACjC,MAAM,YAAY,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,aAAa,GAAG,UAAU,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAE1F,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,UAAU,EAAE,YAAY,EAAE,CAAC;AAC7D,CAAC"}

package/dist/eval-core/layer-gates.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"layer-gates.d.ts","sourceRoot":"","sources":["../../src/eval-core/layer-gates.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAExD,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB;AAED;;;;;;;;;GASG;AACH,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,EACvC,SAAS,EAAE,MAAM,GAChB,eAAe,CAgCjB"}

package/dist/eval-core/layer-gates.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"layer-gates.js","sourceRoot":"","sources":["../../src/eval-core/layer-gates.ts"],"names":[],"mappings":"AAOA;;;;;;;;;GASG;AACH,MAAM,UAAU,kBAAkB,CAChC,OAAuC,EACvC,SAAiB;IAEjB,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,CAAC;QAC7D,MAAM,MAAM,GAAwD;YAClE,EAAE,KAAK,EAAE,WAAW,EAAS,KAAK,EAAE,KAAK,CAAC,YAAY,EAAE;YACxD,EAAE,KAAK,EAAE,eAAe,EAAK,KAAK,EAAE,KAAK,CAAC,gBAAgB,EAAE;YAC5D,EAAE,KAAK,EAAE,gBAAgB,EAAI,KAAK,EAAE,KAAK,CAAC,aAAa,EAAE;SAC1D,CAAC;QACF,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC;QAElE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,SAAS,OAAO,+FAA+F,CAAC,CAAC;YAC5H,OAAO,GAAG,KAAK,CAAC;YAChB,SAAS;QACX,CAAC;QAED,IAAI,WAAW,GAAG,IAAI,CAAC;QACvB,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;YACvB,MAAM,IAAI,GAAG,CAAC,IAAI,SAAS,CAAC;YAC5B,IAAI,CAAC,IAAI;gBAAE,WAAW,GAAG,KAAK,CAAC;YAC/B,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9D,CAAC;QACD,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;QAC7C,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,KAAK,OAAO,MAAM,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,eAAe,SAAS,GAAG,CAAC,CAAC;QACpF,IAAI,CAAC,WAAW;YAAE,OAAO,GAAG,KAAK,CAAC;IACpC,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;AAC5B,CAAC"}

package/dist/eval-core/mock-hook.cjs

@@ -0,0 +1,215 @@
+#!/usr/bin/env node
+/**
+ * omk-mock PreToolUse hook (CLI fallback path).
+ *
+ * 由 src/eval-core/mocks-runtime.ts:materializeForCliConfigDir 写到临时
+ * CLAUDE_CONFIG_DIR/mock-hook.cjs。Claude Code 子进程会以 hook 形式调它,
+ * stdin 收 PreToolUseHookInput,stdout 出 PreToolUseHookOutput JSON。
+ *
+ * 该脚本是无状态的 — 所有规则从 OMK_MOCKS_FILE 文件读。命中状态(状态机用)
+ * 通过同目录下的 hits.json 持久化(同一 spawn 内多次 hook 调用共享)。
+ */
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+
+function expandHome(p) {
+  if (p.startsWith('~/')) return path.join(os.homedir(), p.slice(2));
+  if (p === '~') return os.homedir();
+  return p;
+}
+
+/** 与 mocks-runtime.ts 的 matchesFilePathSuffix 对齐 — 完全相等或路径分隔符后缀。 */
+function matchesFilePathSuffix(actualRaw, suffixRaw) {
+  const actual = expandHome(actualRaw);
+  const suffix = expandHome(suffixRaw);
+  if (actual === suffix) return true;
+  if (!actual.endsWith(suffix)) return false;
+  const sep = actual.charAt(actual.length - suffix.length - 1);
+  return sep === '/' || sep === '\\';
+}
+
+function globMatch(pattern, value) {
+  // 与 SDK 路径(mocks-runtime.ts)对齐:dotAll 让 `*` 跨换行匹配多行 bash。
+  const escaped = pattern.replace(/[.+?^${}()|[\]\\]/g, '\\$&');
+  return new RegExp('^' + escaped.replace(/\*/g, '.*') + '$', 's').test(value);
+}
+
+function arraysDeepEqual(a, b) {
+  if (a.length !== b.length) return false;
+  for (let i = 0; i < a.length; i++) if (!deepEqual(a[i], b[i])) return false;
+  return true;
+}
+function deepEqual(a, b) {
+  if (a === b) return true;
+  if (Array.isArray(a) && Array.isArray(b)) return arraysDeepEqual(a, b);
+  if (typeof a === 'object' && a !== null && typeof b === 'object' && b !== null
+      && !Array.isArray(a) && !Array.isArray(b)) {
+    const ak = Object.keys(a), bk = Object.keys(b);
+    if (ak.length !== bk.length) return false;
+    for (const k of ak) if (!deepEqual(a[k], b[k])) return false;
+    return true;
+  }
+  return false;
+}
+function matchesInputSubset(expected, actual) {
+  if (typeof actual !== 'object' || actual === null) return false;
+  for (const k of Object.keys(expected)) {
+    const v = expected[k];
+    if (Array.isArray(v)) {
+      if (!Array.isArray(actual[k]) || !arraysDeepEqual(v, actual[k])) return false;
+    } else if (typeof v === 'object' && v !== null) {
+      if (!matchesInputSubset(v, actual[k])) return false;
+    } else if (actual[k] !== v) {
+      return false;
+    }
+  }
+  return true;
+}
+
+function anyStringContains(obj, needle) {
+  if (typeof obj === 'string') return obj.toLowerCase().includes(needle);
+  if (Array.isArray(obj)) return obj.some(function(item) { return anyStringContains(item, needle); });
+  if (typeof obj === 'object' && obj !== null) {
+    return Object.values(obj).some(function(v) { return anyStringContains(v, needle); });
+  }
+  return false;
+}
+
+function isMockHit(mock, toolName, toolInput) {
+  if (mock.tool !== '*' && mock.tool !== toolName) return false;
+  const m = mock.match;
+  if (!m) return true;
+  const ti = toolInput || {};
+  if (m.file_path !== undefined) {
+    if (typeof ti.file_path !== 'string') return false;
+    if (expandHome(ti.file_path) !== expandHome(m.file_path)) return false;
+  }
+  if (m.file_path_endswith !== undefined) {
+    if (typeof ti.file_path !== 'string') return false;
+    if (!matchesFilePathSuffix(ti.file_path, m.file_path_endswith)) return false;
+  }
+  if (m.url !== undefined && ti.url !== m.url) return false;
+  if (m.url_glob !== undefined) {
+    if (typeof ti.url !== 'string' || !globMatch(m.url_glob, ti.url)) return false;
+  }
+  if (m.command_glob !== undefined) {
+    if (typeof ti.command !== 'string' || !globMatch(m.command_glob, ti.command)) return false;
+  }
+  if (m.input !== undefined && !matchesInputSubset(m.input, toolInput)) return false;
+  if (m.input_contains !== undefined) {
+    if (!anyStringContains(toolInput, m.input_contains.toLowerCase())) return false;
+  }
+  return true;
+}
+
+function stringifyReturn(r) {
+  if (typeof r === 'string') return r;
+  return JSON.stringify(r);
+}
+
+function resolveMockReturn(mock, hitCount, baseDir) {
+  if (mock.return_seq && hitCount < mock.return_seq.length) {
+    return stringifyReturn(mock.return_seq[hitCount]);
+  }
+  if (mock.return !== undefined) return stringifyReturn(mock.return);
+  if (mock.return_file) {
+    const fpath = path.isAbsolute(mock.return_file)
+      ? mock.return_file
+      : path.resolve(baseDir || process.cwd(), mock.return_file);
+    if (!fs.existsSync(fpath)) return `[omk-mock-error] return_file not found: ${fpath}`;
+    return fs.readFileSync(fpath, 'utf8');
+  }
+  return '';
+}
+
+// ─── main ────────────────────────────────────────────────────────────────────
+
+let stdin = '';
+process.stdin.on('data', (chunk) => { stdin += chunk; });
+process.stdin.on('end', () => {
+  let input;
+  try {
+    input = JSON.parse(stdin);
+  } catch (e) {
+    process.stdout.write(JSON.stringify({ continue: true }));  // hook 协议错误时放行
+    process.exit(0);
+  }
+
+  const mocksFile = process.env.OMK_MOCKS_FILE;
+  if (!mocksFile || !fs.existsSync(mocksFile)) {
+    // 没配 mocks 就 noop
+    process.stdout.write(JSON.stringify({ continue: true }));
+    process.exit(0);
+  }
+
+  const config = JSON.parse(fs.readFileSync(mocksFile, 'utf8'));
+  const mocks = config.mocks || [];
+  const baseDir = config.baseDir;
+  const strict = !!config.strict;
+
+  // 统计文件:perMock(每条 mock 的命中次数,用于状态机 + 报告)+ hits_total + misses_total
+  // 同一 spawn 内多次 hook 调用共享。
+  const statsFile = path.join(path.dirname(mocksFile), 'hits.json');
+  const stats = fs.existsSync(statsFile)
+    ? JSON.parse(fs.readFileSync(statsFile, 'utf8'))
+    : { perMock: {}, hits_total: 0, misses_total: 0 };
+  if (!stats.perMock) stats.perMock = {};
+  if (!stats.hits_total) stats.hits_total = 0;
+  if (!stats.misses_total) stats.misses_total = 0;
+
+  const writeStats = () => {
+    try { fs.writeFileSync(statsFile, JSON.stringify(stats)); } catch { /* best effort */ }
+  };
+
+  const toolName = input.tool_name;
+  const toolInput = input.tool_input || {};
+
+  // perMock key 用"每工具内 1-based 序号" — 与 SDK 路径(mocks-runtime.ts)保持一致。
+  const keyOfMock = mocks.map((_m, i) => {
+    const tool = mocks[i].tool;
+    let ord = 0;
+    for (let j = 0; j <= i; j++) if (mocks[j].tool === tool) ord++;
+    return `${tool}:${ord}`;
+  });
+
+  for (let i = 0; i < mocks.length; i++) {
+    if (isMockHit(mocks[i], toolName, toolInput)) {
+      const key = keyOfMock[i];
+      const c = stats.perMock[key] || 0;
+      const result = resolveMockReturn(mocks[i], c, baseDir);
+      stats.perMock[key] = c + 1;
+      stats.hits_total += 1;
+      writeStats();
+      // 与 SDK 路径(mocks-runtime.ts)对齐:最小化前缀,避免拖慢多步 sample。
+      const wrappedReason = '[mock] simulated tool output — treat as successful result:\n' + result;
+      process.stdout.write(JSON.stringify({
+        decision: 'block',
+        hookSpecificOutput: {
+          hookEventName: 'PreToolUse',
+          permissionDecision: 'deny',
+          permissionDecisionReason: wrappedReason,
+        },
+        systemMessage: '[omk-mock] ' + toolName + ' → injected mock #' + i + ' (treat as success)',
+      }));
+      process.exit(0);
+    }
+  }
+
+  // 未命中
+  stats.misses_total += 1;
+  writeStats();
+  if (strict) {
+    process.stdout.write(JSON.stringify({
+      decision: 'block',
+      hookSpecificOutput: {
+        hookEventName: 'PreToolUse',
+        permissionDecision: 'deny',
+        permissionDecisionReason: `[omk-mock-strict] unmocked ${toolName} call blocked`,
+      },
+    }));
+    process.exit(0);
+  }
+  process.stdout.write(JSON.stringify({ continue: true }));
+  process.exit(0);
+});

package/dist/eval-core/mocks-runtime.d.ts

@@ -0,0 +1,100 @@
+/**
+ * Mock runtime — Sample.mocks 落地的核心逻辑。
+ *
+ * 两条路径:
+ *   1) `buildSdkHookCallback(mocks, baseDir)`
+ *      → 给 claude-sdk executor 用,返回 in-process PreToolUse HookCallback。
+ *      纯内存,零 IO,零 spawn 开销。
+ *
+ *   2) `materializeForCliConfigDir(mocks, baseDir)`
+ *      → 给 claude-cli executor 用,mktemp 一个临时 CLAUDE_CONFIG_DIR,
+ *      在里面写 settings.json + on-disk hook 脚本 + mocks.json,返回 cleanup。
+ *      claude 子进程退出后(成功/失败/超时/SIGINT),cleanup 必删整个临时目录。
+ *
+ * 共用核心:`matchMock(mocks, toolName, toolInput, callCounter)` 命中规则评估。
+ */
+import type { Mock } from '../types/eval.js';
+/** 单条 mock 是否命中给定 tool 调用。 */
+export declare function isMockHit(mock: Mock, toolName: string, toolInput: unknown): boolean;
+/** 把 mock 的 return / return_file / return_seq 解析成可序列化字符串(LLM 看到的 tool_result)。 */
+export declare function resolveMockReturn(mock: Mock, hitCount: number, baseDir?: string): string;
+/** SDK PreToolUse hook input(精简版,只用我们关心的字段)。 */
+interface SdkHookInput {
+    hook_event_name: 'PreToolUse';
+    tool_name: string;
+    tool_input: unknown;
+}
+/** SDK PreToolUse hook output:`block` 决策 + permissionDecisionReason 当 mock content。 */
+interface SdkHookOutput {
+    continue?: boolean;
+    decision?: 'block' | 'approve';
+    hookSpecificOutput?: {
+        hookEventName: 'PreToolUse';
+        permissionDecision?: 'deny' | 'allow' | 'ask';
+        permissionDecisionReason?: string;
+    };
+    systemMessage?: string;
+}
+export interface SdkHookHandle {
+    /** 给 SDK options.hooks.PreToolUse 用的 callback。 */
+    callback: (input: SdkHookInput) => Promise<SdkHookOutput>;
+    /** 拦截命中统计(诊断用)。 */
+    stats: {
+        hits: number;
+        misses: number;
+        perMock: Record<string, number>;
+    };
+}
+/**
+ * 把 mocks 转成 SDK in-process PreToolUse hook callback。
+ *
+ * 命中策略:strict 模式下不命中的工具调用直接 deny(防意外真调)。
+ * 默认非 strict:不命中放行(原生工具继续真跑)— 适合"部分 mock,部分真跑"场景。
+ *
+ * @param mocks  Sample.mocks 数组
+ * @param baseDir  解析 mock.return_file 的相对路径锚点
+ * @param strict  true → 不命中 deny;false → 不命中放行(default)
+ */
+export declare function buildSdkHookCallback(mocks: Mock[] | undefined, baseDir?: string, strict?: boolean): SdkHookHandle;
+export interface CliMockHandle {
+    /** 临时 settings 文件路径,作为 `claude --settings <path>` 参数传入。
+     *  Claude Code 会**追加**这个 settings 到 ~/.claude/settings.json,**不替换** —
+     *  这样 OAuth 登录态 / 用户主配置都不动。 */
+    settingsFile: string;
+    /** 临时 MCP 配置文件路径,作为 `claude --mcp-config <path> --strict-mcp-config` 参数传入。 */
+    mcpConfigFile?: string;
+    /** 子进程要看到的额外 env(目前只 OMK_MOCKS_FILE)。 */
+    env: Record<string, string>;
+    /** 读 hook 在临时目录写的命中统计(给 ExecResult.mockStats 用)。
+     *  必须在 cleanup 之前调,cleanup 后文件就没了。 */
+    readStats: () => {
+        hits: number;
+        misses: number;
+        perMock: Record<string, number>;
+    };
+    /** 必须在 spawn 完成后(无论 success/error/timeout)调用。 */
+    cleanup: () => void;
+}
+/**
+ * 物化 mocks 到临时 settings 文件 + on-disk hook 脚本。
+ *
+ * **关键**:用 Claude Code 的 `--settings <file>` flag 而非 `CLAUDE_CONFIG_DIR`。
+ * `--settings` 是**追加**模式,主 `~/.claude/` 包含的 OAuth 登录态 / 用户配置全部保留;
+ * 而 CLAUDE_CONFIG_DIR 是**替换**模式,设它会丢 OAuth → "Not logged in"。
+ *
+ * 临时目录结构:
+ *   $tmpdir/omk-mocks-XXXXXX/
+ *     ├── settings.json       (PreToolUse hook 注册,作为 --settings 参数)
+ *     ├── mock-hook.cjs       (hook 实现:读 OMK_MOCKS_FILE → 匹配 → 输出 JSON 决策)
+ *     ├── fake-mcp-server.cjs (可选:给 mcp__server__tool mock 注册一次性 fake MCP)
+ *     ├── mcp.json            (可选:作为 --mcp-config 参数)
+ *     └── mocks.json          (mocks 序列化 + baseDir + strict)
+ *
+ * cleanup 必删整个目录。executor 用 try / finally 保证执行。
+ */
+export declare function materializeForCliConfigDir(mocks: Mock[] | undefined, baseDir?: string, strict?: boolean): CliMockHandle | null;
+declare function readMockHookTemplate(): string;
+export declare const _readMockHookTemplateForTest: typeof readMockHookTemplate;
+export declare function _testMakeTempConfigDir(): string;
+export {};
+//# sourceMappingURL=mocks-runtime.d.ts.map

package/dist/eval-core/mocks-runtime.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mocks-runtime.d.ts","sourceRoot":"","sources":["../../src/eval-core/mocks-runtime.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAMH,OAAO,KAAK,EAAE,IAAI,EAAc,MAAM,kBAAkB,CAAC;AAqFzD,8BAA8B;AAC9B,wBAAgB,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,GAAG,OAAO,CA8BnF;AAID,kFAAkF;AAClF,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,IAAI,EACV,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,MAAM,GACf,MAAM,CAkBR;AAWD,gDAAgD;AAChD,UAAU,YAAY;IACpB,eAAe,EAAE,YAAY,CAAC;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,OAAO,CAAC;CACrB;AAED,uFAAuF;AACvF,UAAU,aAAa;IACrB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,QAAQ,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC/B,kBAAkB,CAAC,EAAE;QACnB,aAAa,EAAE,YAAY,CAAC;QAC5B,kBAAkB,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,KAAK,CAAC;QAC9C,wBAAwB,CAAC,EAAE,MAAM,CAAC;KACnC,CAAC;IACF,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B,kDAAkD;IAClD,QAAQ,EAAE,CAAC,KAAK,EAAE,YAAY,KAAK,OAAO,CAAC,aAAa,CAAC,CAAC;IAC1D,mBAAmB;IACnB,KAAK,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,CAAC;CAC1E;AAED;;;;;;;;;GASG;AACH,wBAAgB,oBAAoB,CAClC,KAAK,EAAE,IAAI,EAAE,GAAG,SAAS,EACzB,OAAO,CAAC,EAAE,MAAM,EAChB,MAAM,UAAQ,GACb,aAAa,CAqDf;AAID,MAAM,WAAW,aAAa;IAC5B;;mCAE+B;IAC/B,YAAY,EAAE,MAAM,CAAC;IACrB,8EAA8E;IAC9E,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,yCAAyC;IACzC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC5B;0CACsC;IACtC,SAAS,EAAE,MAAM;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,CAAC;IACnF,iDAAiD;IACjD,OAAO,EAAE,MAAM,IAAI,CAAC;CACrB;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,0BAA0B,CACxC,KAAK,EAAE,IAAI,EAAE,GAAG,SAAS,EACzB,OAAO,CAAC,EAAE,MAAM,EAChB,MAAM,UAAQ,GACb,aAAa,GAAG,IAAI,CA0EtB;AA+MD,iBAAS,oBAAoB,IAAI,MAAM,CAatC;AAGD,eAAO,MAAM,4BAA4B,6BAAuB,CAAC;AAIjE,wBAAgB,sBAAsB,IAAI,MAAM,CAI/C"}