oh-my-knowledge 0.48.0 → 0.49.0

package/dist/eval-core/execution-strategy.js

@@ -1,5 +1,4 @@
 import { dirname } from 'node:path';
-import { existsSync, mkdirSync } from 'node:fs';
 import { DEFAULT_ISOLATED_CWD_DIR } from './default-dirs.js';
 /**
  * Isolated cwd for strict baseline. Empty dir under ~/.oh-my-knowledge/
@@ -8,14 +7,11 @@ import { DEFAULT_ISOLATED_CWD_DIR } from './default-dirs.js';
  * leak the skill content into baseline via plain file-system access, even when
  * Skill auto-discovery is blocked).
  *
- * Stable path (not per-task tmpdir) so cache keys remain consistent.
+ * This is a stable logical path for cache identity. executeTasks replaces it
+ * with a fresh physical directory for every executor attempt, then removes it.
  */
 function getIsolatedCwd() {
-    const dir = DEFAULT_ISOLATED_CWD_DIR;
-    if (!existsSync(dir)) {
-        mkdirSync(dir, { recursive: true });
-    }
-    return dir;
+    return DEFAULT_ISOLATED_CWD_DIR;
 }
 function combineUserPrompt(prefix, prompt) {
     if (!prefix)
@@ -84,6 +80,11 @@ function extractSkillDir(artifact) {
     return dirname(artifact.locator);
 }
 export function resolveExecutionStrategy(task, model, timeoutMs, verbose, effort, samplesBaseDir) {
+    if (Array.isArray(task.artifact.allowedSkills)
+        && task.artifact.allowedSkills.length > 0) {
+        throw new Error(`skill 白名单(allowedSkills=${JSON.stringify(task.artifact.allowedSkills)})不再支持：`
+            + '非空白名单无法在所有执行器上保证隔离。请使用 [](完全隔离)或省略该字段。');
+    }
     const skillDir = extractSkillDir(task.artifact);
     // strict-baseline cwd 沙箱:baseline + allowedSkills===[] + 用户没显式
     // cwd 时,改用 isolated empty dir。否则 baseline 的 Glob/Read 会走进用户工作目录
@@ -102,8 +103,8 @@ export function resolveExecutionStrategy(task, model, timeoutMs, verbose, effort
         ...(effort && { effort }),
         // pass skill-isolation declaration to executors. undefined keeps
         // SDK default; [] = strict isolation (skills:[] + disallowedTools:['Skill']).
-        // non-empty allowedSkills is rejected upstream (validateEvalConfig) and by every
-        // executor — a skill whitelist could not be fully isolated, so it was removed.
+        // Non-empty allowedSkills is rejected at this common planning boundary, including
+        // programmatic callers that bypass eval.yaml validation.
         ...(task.artifact.allowedSkills !== undefined && { allowedSkills: task.artifact.allowedSkills }),
         // Sample.mocks 透传到 executor。executor(claude-sdk / claude-cli)
         // 自决定怎么落地(in-process hook vs 临时 CLAUDE_CONFIG_DIR + on-disk hook)。
@@ -121,6 +122,7 @@ export function resolveExecutionStrategy(task, model, timeoutMs, verbose, effort
             return {
                 strategy: resolveArtifactExecutionStrategy(task.artifact),
                 cacheSystem: '',
+                ...(isStrictBaseline && !task.cwd && { isolatedCwd: true }),
                 input: {
                     ...baseInput,
                     system: null,

package/dist/eval-core/fact-checker.js

@@ -3,15 +3,15 @@
  * Currently supports file/directory path verification via fs.existsSync.
  */
 import { existsSync } from 'node:fs';
-import { resolve } from 'node:path';
+import { isAbsolute, relative, resolve } from 'node:path';
 // Patterns that look like file/directory paths in agent output
 const PATH_PATTERNS = [
     // Explicit code paths: repos/xxx, src/xxx, lib/xxx, packages/xxx
     /(?:repos|src|lib|packages|dist|test|components)\/[a-zA-Z0-9_/.@-]+/g,
-    // .claude paths
-    /\.claude\/[a-zA-Z0-9_/.@-]+/g,
+    // Agent configuration and skill paths across supported trace sources.
+    /\.(?:agents|claude|codex|gemini)\/[a-zA-Z0-9_/.@-]+/g,
     // Paths with file extensions mentioned in text
-    /[a-zA-Z0-9_/-]+\.(?:ts|js|tsx|jsx|md|json|yaml|yml|sh)\b(?!\s*\()/g,
+    /(?<![a-zA-Z0-9_./-])[a-zA-Z0-9_/-]+\.(?:ts|js|tsx|jsx|md|json|yaml|yml|sh)\b(?!\s*\()/g,
 ];
 // Paths to ignore (too generic or always present)
 const IGNORE_PATTERNS = [
@@ -48,14 +48,22 @@ export function extractPathClaims(output) {
  */
 export function checkFacts(output, cwd) {
     const pathClaims = extractPathClaims(output);
+    const root = resolve(cwd);
     const claims = pathClaims.map((path) => {
-        const fullPath = resolve(cwd, path);
-        const exists = existsSync(fullPath);
+        const fullPath = resolve(root, path);
+        const relativePath = relative(root, fullPath);
+        const insideCwd = relativePath === ''
+            || (!relativePath.startsWith('..') && !isAbsolute(relativePath));
+        const exists = insideCwd && existsSync(fullPath);
         return {
             type: 'file-path',
             value: path,
             verified: exists,
-            ...(!exists && { evidence: `${fullPath} not found` }),
+            ...(!exists && {
+                evidence: insideCwd
+                    ? `${fullPath} not found`
+                    : `${path} is outside the evaluation cwd`,
+            }),
         };
     });
     const verifiedCount = claims.filter((c) => c.verified).length;

package/dist/eval-core/holdout.js

@@ -12,6 +12,7 @@
  * signal the verdict's overfitting gate reads (`src/eval-core/verdict.ts`).
  */
 import { buildVariantSummary } from './schema.js';
+import { setOwnRecordValue } from '../shared/record-count.js';
 /** Below this many samples on any side, a split is too small to be meaningful —
  *  callers fall back to full-set scoring and mark the breakdown `disabled`. */
 export const MIN_HOLDOUT_SUBSET = 3;
@@ -105,14 +106,14 @@ export function computeHoldoutBreakdown(report, variantNames, ratio, sampleIdOrd
     }
     const perVariant = {};
     for (const v of variantNames) {
-        perVariant[v] = {
+        setOwnRecordValue(perVariant, v, {
             trainScore: Number(subsetCompositeScore(report, v, split.trainIds).toFixed(4)),
             holdoutScore: Number(subsetCompositeScore(report, v, split.holdoutIds).toFixed(4)),
             trainCount: split.trainIds.size,
             holdoutCount: split.holdoutIds.size,
             trainScorable: subsetScorableCount(report, v, split.trainIds),
             holdoutScorable: subsetScorableCount(report, v, split.holdoutIds),
-        };
+        });
     }
     return { ratio, perVariant };
 }

package/dist/eval-core/judge-independence.d.ts

@@ -3,8 +3,8 @@ import { type ExecutorVendor } from '../executors/shared.js';
 /**
  * 评委独立性分析(单一来源,verdict caveat 与 analysis 诊断共用)。
  *
- * LLM 评委有自我偏好偏置:偏爱与自己同模型家族产出的输出。omk 默认评委(claude:haiku)与默认
- * 执行器(claude:*)同属一家,敞口默认就开着。本 helper 只算客观事实(评委/输出各属哪家、有没有
+ * LLM 评委有自我偏好偏置:偏爱与自己同模型家族产出的输出。CLI 默认评委跟随所选 runtime
+ * (Claude 用 haiku,Codex 沿用被测模型),故敞口默认就开着。本 helper 只算客观事实(评委/输出各属哪家、有没有
  * 跨厂商评委、是不是单厂商 ensemble、有没有挂 gold 校准),严重度与文案交给消费方(report-diagnostics
  * / verdict)决定。
  *

package/dist/eval-core/mock-hook.cjs

@@ -13,6 +13,22 @@ const fs = require('fs');
 const path = require('path');
 const os = require('os');
 
+function recordCount(record, key) {
+  const value = Object.prototype.hasOwnProperty.call(record, key) ? record[key] : undefined;
+  return typeof value === 'number' && Number.isInteger(value) && value >= 0 ? value : 0;
+}
+
+function incrementRecordCount(record, key, amount = 1) {
+  const next = recordCount(record, key) + amount;
+  Object.defineProperty(record, key, {
+    value: next,
+    enumerable: true,
+    configurable: true,
+    writable: true,
+  });
+  return next;
+}
+
 function expandHome(p) {
   if (p.startsWith('~/')) return path.join(os.homedir(), p.slice(2));
   if (p === '~') return os.homedir();
@@ -151,12 +167,13 @@ process.stdin.on('end', () => {
   // 统计文件:perMock(每条 mock 的命中次数,用于状态机 + 报告)+ hits_total + misses_total
   // 同一 spawn 内多次 hook 调用共享。
   const statsFile = path.join(path.dirname(mocksFile), 'hits.json');
-  const stats = fs.existsSync(statsFile)
+  let stats = fs.existsSync(statsFile)
     ? JSON.parse(fs.readFileSync(statsFile, 'utf8'))
     : { perMock: {}, hits_total: 0, misses_total: 0 };
-  if (!stats.perMock) stats.perMock = {};
-  if (!stats.hits_total) stats.hits_total = 0;
-  if (!stats.misses_total) stats.misses_total = 0;
+  if (!stats || typeof stats !== 'object' || Array.isArray(stats)) stats = {};
+  if (!stats.perMock || typeof stats.perMock !== 'object' || Array.isArray(stats.perMock)) stats.perMock = {};
+  stats.hits_total = recordCount(stats, 'hits_total');
+  stats.misses_total = recordCount(stats, 'misses_total');
 
   const writeStats = () => {
     try { fs.writeFileSync(statsFile, JSON.stringify(stats)); } catch { /* best effort */ }
@@ -176,9 +193,9 @@ process.stdin.on('end', () => {
   for (let i = 0; i < mocks.length; i++) {
     if (isMockHit(mocks[i], toolName, toolInput)) {
       const key = keyOfMock[i];
-      const c = stats.perMock[key] || 0;
+      const c = recordCount(stats.perMock, key);
       const result = resolveMockReturn(mocks[i], c, baseDir);
-      stats.perMock[key] = c + 1;
+      incrementRecordCount(stats.perMock, key);
       stats.hits_total += 1;
       writeStats();
       // 与 SDK 路径(mocks-runtime.ts)对齐:最小化前缀,避免拖慢多步 sample。

package/dist/eval-core/mocks-runtime.js

@@ -17,6 +17,7 @@ import { mkdtempSync, rmSync, writeFileSync, mkdirSync, readFileSync, existsSync
 import { homedir, tmpdir } from 'node:os';
 import { dirname, isAbsolute, join, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
+import { incrementRecordCount, setOwnRecordValue } from '../shared/record-count.js';
 // ─── Match logic ────────────────────────────────────────────────────────────
 function expandHome(p) {
     if (p.startsWith('~/'))
@@ -209,7 +210,7 @@ export function buildSdkHookCallback(mocks, baseDir, strict = false) {
                 hitCounters.set(i, c + 1);
                 stats.hits++;
                 const key = keyOfMock[i];
-                stats.perMock[key] = (stats.perMock[key] || 0) + 1;
+                incrementRecordCount(stats.perMock, key);
                 // 关键 UX:LLM 看到 permissionDecision='deny' 容易误判"被拒绝/失败"。
                 // 给 reason 加一行最小前缀,告诉它把内容当成真实成功输出。措辞越短越好 ——
                 // 长措辞会显著拉慢多步 sample(每个工具调用都要读这段),实测 6 步流程从 60s 拖到 120s+。
@@ -289,11 +290,11 @@ export function materializeForCliConfigDir(mocks, baseDir, strict = false) {
         writeFileSync(mcpServerScript, fakeMcpServerSource(), 'utf8');
         const mcpServers = {};
         for (const serverName of fakeMcpServers.keys()) {
-            mcpServers[serverName] = {
+            setOwnRecordValue(mcpServers, serverName, {
                 command: 'node',
                 args: [mcpServerScript, serverName],
                 env: { OMK_MOCKS_FILE: mocksFile },
-            };
+            });
         }
         writeFileSync(mcpConfigFile, JSON.stringify({ mcpServers }, null, 2));
     }
@@ -359,6 +360,20 @@ const mocksFile = process.env.OMK_MOCKS_FILE;
 const rl = readline.createInterface({ input: process.stdin });
 
 function send(obj) { process.stdout.write(JSON.stringify(obj) + '\\n'); }
+function recordCount(record, key) {
+  const value = Object.prototype.hasOwnProperty.call(record, key) ? record[key] : undefined;
+  return typeof value === 'number' && Number.isInteger(value) && value >= 0 ? value : 0;
+}
+function incrementRecordCount(record, key, amount = 1) {
+  const next = recordCount(record, key) + amount;
+  Object.defineProperty(record, key, {
+    value: next,
+    enumerable: true,
+    configurable: true,
+    writable: true,
+  });
+  return next;
+}
 function globMatch(pattern, value) {
   const escaped = pattern.replace(/[.+?^\${}()|[\\]\\\\]/g, '\\\\$&');
   return new RegExp('^' + escaped.replace(/\\*/g, '.*') + '$', 's').test(value);
@@ -445,7 +460,12 @@ function schemaForTool(name, mocks) {
     const input = m.match && m.match.input;
     if (typeof input !== 'object' || input === null || Array.isArray(input)) continue;
     for (const k of Object.keys(input)) {
-      properties[k] = inferJsonSchema(input[k]);
+      Object.defineProperty(properties, k, {
+        value: inferJsonSchema(input[k]),
+        enumerable: true,
+        configurable: true,
+        writable: true,
+      });
       if (!required.includes(k)) required.push(k);
     }
   }
@@ -476,9 +496,10 @@ function recordHit(mockKey) {
   const statsFile = path.join(path.dirname(mocksFile), 'hits.json');
   let stats = { perMock: {}, hits_total: 0, misses_total: 0 };
   try { if (fs.existsSync(statsFile)) stats = JSON.parse(fs.readFileSync(statsFile, 'utf8')); } catch {}
-  if (!stats.perMock) stats.perMock = {};
-  stats.perMock[mockKey] = (stats.perMock[mockKey] || 0) + 1;
-  stats.hits_total = (stats.hits_total || 0) + 1;
+  if (!stats || typeof stats !== 'object' || Array.isArray(stats)) stats = {};
+  if (!stats.perMock || typeof stats.perMock !== 'object' || Array.isArray(stats.perMock)) stats.perMock = {};
+  incrementRecordCount(stats.perMock, mockKey);
+  stats.hits_total = recordCount(stats, 'hits_total') + 1;
   fs.writeFileSync(statsFile, JSON.stringify(stats));
 }
 function recordMiss() {
@@ -486,7 +507,8 @@ function recordMiss() {
   const statsFile = path.join(path.dirname(mocksFile), 'hits.json');
   let stats = { perMock: {}, hits_total: 0, misses_total: 0 };
   try { if (fs.existsSync(statsFile)) stats = JSON.parse(fs.readFileSync(statsFile, 'utf8')); } catch {}
-  stats.misses_total = (stats.misses_total || 0) + 1;
+  if (!stats || typeof stats !== 'object' || Array.isArray(stats)) stats = {};
+  stats.misses_total = recordCount(stats, 'misses_total') + 1;
   fs.writeFileSync(statsFile, JSON.stringify(stats));
 }
 const hitCounters = new Map();

package/dist/eval-core/report-document.d.ts

@@ -0,0 +1,12 @@
+import type { ReportDocument, ReportIndexCard } from '../types/index.js';
+export declare function isCanonicalReportId(value: unknown): value is string;
+/**
+ * Parse a persisted report at every storage boundary. Canonical report files
+ * are the source of truth; index cards must never relax this contract.
+ */
+export declare function parseReportDocument(value: unknown, fallbackId: string, expectedId?: string): ReportDocument | null;
+/**
+ * Validate a lightweight report discovery card. The card is still only a
+ * pointer: consumers must parse `path` through `parseReportDocument` before use.
+ */
+export declare function parseReportIndexCard(value: unknown): ReportIndexCard | null;