oh-my-knowledge 0.40.0 → 0.42.0

package/dist/cli/commands/eval/index.js

@@ -8,6 +8,7 @@ import { parseRunConfig } from '../../lib/parse-run-config.js';
 import { makeOnProgress } from '../../lib/progress.js';
 import { computeRunTally } from '../../lib/run-tally.js';
 import { DEFAULT_BOOTSTRAP_SAMPLES } from '../../../eval-core/bootstrap.js';
+import { DEFAULT_GATE_THRESHOLD } from '../../../eval-core/verdict.js';
 import { EVALUATION_REPORT_SCHEMA_VERSION } from '../../../eval-core/evaluation-reporting.js';
 function isDryRunReport(report) {
     return Boolean(report && typeof report === 'object' && report.dryRun === true);
@@ -17,9 +18,11 @@ function isDryRunBatchReport(report) {
 }
 function verdictOptions(values) {
     const rawThreshold = values.threshold;
+    // 不传 --threshold 时返回 undefined,由 computeVerdict 应用 DEFAULT_GATE_THRESHOLD ——
+    // 避免在此再硬编码一份 3.5(单一来源在 verdict.ts)。
     const gateThreshold = rawThreshold !== undefined && Number.isFinite(Number(rawThreshold))
         ? Number(rawThreshold)
-        : 3.5;
+        : undefined;
     const rawTrivial = values['trivial-diff'];
     const triviallySmallDiff = rawTrivial !== undefined && Number.isFinite(Number(rawTrivial))
         ? Number(rawTrivial)
@@ -38,10 +41,30 @@ function applyGateExitCode(code, values, lang) {
     process.stderr.write(tCli('cli.run.report_only_gate_skipped', lang));
     return 0;
 }
+/**
+ * 完整 report JSON 是**机器输出**:重定向 / 管道(`omk eval > r.json`、`| jq`)时吐到 stdout 供下游消费。
+ * 交互式 TTY 下报告已存盘、(默认)还起了 report server,再刷上千行 JSON 只会把 verdict 淹没在屏幕外 ——
+ * 故只在非 TTY(stdout 被重定向 / 管道)时 dump。dry-run 的 JSON 是用户显式索取的产物,不走此门控。
+ */
+function emitReportJson(report) {
+    if (!process.stdout.isTTY) {
+        console.log(JSON.stringify(report, null, 2));
+    }
+}
+/**
+ * 给人读的 verdict 文案:stdout 是 TTY 时进 stdout(交互终端没有 JSON,verdict 就是答案),
+ * 否则进 stderr —— 与 emitReportJson 配对,保证非 TTY 的 stdout 是**纯 report JSON**,
+ * `omk eval | jq` / `> report.json` 不会被末尾拼上的人类文案噎住(否则 JSON.parse 直接失败)。
+ */
+function emitVerdictText(text) {
+    // 与 console.log 等价(对单个字符串 = write(text + '\n')),只切换目标流,逐字节保留既有文案。
+    const stream = process.stdout.isTTY ? process.stdout : process.stderr;
+    stream.write(text + '\n');
+}
 async function emitEvaluationVerdict(report, values, lang) {
     const { computeVerdict, formatVerdictText } = await import('../../../eval-core/verdict.js');
     const result = computeVerdict(report, verdictOptions(values));
-    console.log(formatVerdictText(result, { verbose: true }));
+    emitVerdictText(formatVerdictText(result, { verbose: true, lang }));
     await recordEvidenceSafely(report, result.level, values, lang);
     return verdictPasses(result.level, result.headline) ? 0 : 1;
 }
@@ -120,13 +143,13 @@ async function emitBatchVerdict(report, reportsDir, values, lang) {
     const status = lang === 'zh'
         ? (failed === 0 ? '通过' : '未通过')
         : (failed === 0 ? 'PASS' : 'FAIL');
-    console.log(tCli('cli.run.batch_verdict_header', lang, {
+    emitVerdictText(tCli('cli.run.batch_verdict_header', lang, {
         status,
         passed,
         total: results.length,
     }));
     for (const result of results) {
-        console.log(`  ${result.verdict.level}: ${result.treatment} — ${result.verdict.headline}`);
+        emitVerdictText(`  ${result.verdict.level}: ${result.treatment} — ${result.verdict.headline}`);
     }
     return failed === 0 ? 0 : 1;
 }
@@ -158,9 +181,6 @@ async function announceSavedReport({ report, filePath, reportsDir, values, lang,
 async function runEval(_args, flags, lang) {
     const { values, config, evalConfig } = parseRunConfig({ ...flags });
     const { runEvaluation, runMultiple, runBatchEvaluation } = await import('../../../eval-workflows/run-evaluation.js');
-    if (values.blind !== undefined) {
-        config.blind = values.blind;
-    }
     config.onProgress = makeOnProgress(lang);
     const repeatRaw = values.repeat;
     const parsedRepeat = repeatRaw !== undefined ? Number(repeatRaw) : (evalConfig?.repeat ?? 1);
@@ -168,6 +188,13 @@ async function runEval(_args, flags, lang) {
         process.stderr.write(tCli('cli.run.invalid_repeat', lang, { value: repeatRaw }));
     }
     const repeatCount = Math.max(1, Math.floor(parsedRepeat) || 1);
+    const holdoutRatioRaw = values['holdout-ratio'];
+    const parsedHoldoutRatio = holdoutRatioRaw !== undefined ? Number(holdoutRatioRaw) : (evalConfig?.holdoutRatio ?? 0);
+    if (holdoutRatioRaw !== undefined && (!Number.isFinite(parsedHoldoutRatio) || parsedHoldoutRatio <= 0 || parsedHoldoutRatio >= 1)) {
+        process.stderr.write(tCli('cli.run.invalid_holdout_ratio', lang, { value: holdoutRatioRaw }));
+    }
+    if (parsedHoldoutRatio > 0 && parsedHoldoutRatio < 1)
+        config.holdoutRatio = parsedHoldoutRatio;
     const judgeRepeatRaw = values['judge-repeat'];
     const parsedJudgeRepeat = judgeRepeatRaw !== undefined ? Number(judgeRepeatRaw) : (evalConfig?.judgeRepeat ?? 1);
     if (judgeRepeatRaw !== undefined && (!Number.isFinite(parsedJudgeRepeat) || parsedJudgeRepeat < 1)) {
@@ -224,11 +251,12 @@ async function runEval(_args, flags, lang) {
                     }
                 },
             });
-            console.log(JSON.stringify(report, null, 2));
             if (isDryRunBatchReport(report)) {
+                console.log(JSON.stringify(report, null, 2));
                 console.log(tCli('cli.run.dry_run_no_scores', lang));
                 throw new CliExit(0);
             }
+            emitReportJson(report);
             if (filePath) {
                 await announceSavedReport({ report, filePath, reportsDir: config.outputDir, values, lang });
             }
@@ -285,7 +313,7 @@ async function runEval(_args, flags, lang) {
                 }
             }
         }
-        console.log(JSON.stringify(report, null, 2));
+        emitReportJson(report);
         if (filePath) {
             await announceSavedReport({ report, filePath, reportsDir: config.outputDir, values, lang });
         }
@@ -368,8 +396,8 @@ export default class Eval extends BaseCommand {
         }),
         'judge-models': Flags.string({
             description: bilingual({
-                zh: '评委配置，格式 executor:model[,...]，例 claude:haiku 或 claude:opus,openai:gpt-4o(≥ 2 个 = ensemble）。默认 <executor>:haiku。',
-                en: 'Judge config: executor:model[,...]. e.g. claude:haiku or claude:opus,openai:gpt-4o (≥ 2 = ensemble). Default <executor>:haiku.',
+                zh: '评委配置，格式 executor:model[,...]，例 claude:haiku 或 claude:opus,openai-api:gpt-4o(≥ 2 个 = ensemble）。默认 <executor>:haiku。',
+                en: 'Judge config: executor:model[,...]. e.g. claude:haiku or claude:opus,openai-api:gpt-4o (≥ 2 = ensemble). Default <executor>:haiku.',
             }),
         }),
         'output-dir': Flags.string({
@@ -453,13 +481,17 @@ export default class Eval extends BaseCommand {
             }),
         }),
         // ── eval-runner extra ──
-        blind: Flags.boolean({
-            description: bilingual({ zh: 'judge blind 模式', en: 'Blind judge mode' }),
-        }),
         repeat: Flags.string({
             description: bilingual({ zh: '每个 sample 重复跑 N 次', en: 'Repeat each sample N times' }),
             parse: integerStringParser('--repeat', { min: 1 }),
         }),
+        'holdout-ratio': Flags.string({
+            description: bilingual({
+                zh: '留出比例 0-1（如 0.3）；切出 holdout 子集，对比 train/holdout 综合分检测过拟合',
+                en: 'Holdout fraction 0-1 (e.g. 0.3); splits a holdout subset, compares train/holdout composite to flag overfitting',
+            }),
+            parse: numberStringParser('--holdout-ratio', { min: 0, max: 1 }),
+        }),
         'judge-repeat': Flags.string({
             description: bilingual({ zh: '每个 dim 评 N 次', en: 'Judge each dim N times' }),
             parse: integerStringParser('--judge-repeat', { min: 1 }),
@@ -490,7 +522,10 @@ export default class Eval extends BaseCommand {
             parse: numberStringParser('--budget-per-sample-ms', { minExclusive: 0 }),
         }),
         threshold: Flags.string({
-            description: bilingual({ zh: 'verdict 阈值，默认 3.5', en: 'Verdict threshold, default 3.5' }),
+            description: bilingual({
+                zh: `verdict 阈值，默认 ${DEFAULT_GATE_THRESHOLD}`,
+                en: `Verdict threshold, default ${DEFAULT_GATE_THRESHOLD}`,
+            }),
             parse: numberStringParser('--threshold'),
         }),
         'trivial-diff': Flags.string({

package/dist/cli/commands/init.js

@@ -12,17 +12,21 @@ const INIT_OMK_GITIGNORE = `# omk 测量 bulk + doctor --fix 备份(项目本地
 /reports/
 /backups/
 `;
+// 脚手架用例必须过 omk 自身的断言合规校验(load-samples.ts Rule A),否则新用户照
+// 快速开始跑的第一条 omk eval 会直接硬报错。约束:contains / not_contains 的 value
+// 只能是单个 ASCII token(长度 [2,40]、无内部空白、无 CJK);多词 / 中文语义匹配一律
+// 走 rubric 交评委判;regex pattern 不能含 CJK。改这里前先跑 `omk eval --dry-run`
+// (非 lenient 合规 oracle)与 test/cli/init-scaffold-conformance 回归测试。
 const INIT_SAMPLES = `[
   {
     "sample_id": "s001",
     "prompt": "审查以下代码",
     "context": "function authenticate(username, password) {\\n  const query = \`SELECT * FROM users WHERE name='\${username}' AND pass='\${password}'\`;\\n  return db.execute(query);\\n}",
-    "rubric": "应识别 SQL 注入风险，建议使用参数化查询",
+    "rubric": "应识别 SQL 注入风险，建议使用参数化查询；不应把这段代码判为安全无问题。",
     "assertions": [
       { "type": "contains", "value": "SQL", "weight": 1 },
       { "type": "contains", "value": "injection", "weight": 1 },
-      { "type": "regex", "pattern": "parameterized|prepared|placeholder|bind", "flags": "i", "weight": 0.5 },
-      { "type": "not_contains", "value": "looks good", "weight": 0.5 }
+      { "type": "regex", "pattern": "parameterized|prepared|placeholder|bind", "flags": "i", "weight": 0.5 }
     ],
     "dimensions": {
       "security": "是否准确识别出 SQL 注入漏洞并说明其危害",
@@ -35,8 +39,7 @@ const INIT_SAMPLES = `[
     "context": "async function fetchData(url) {\\n  const res = await fetch(url);\\n  const data = await res.json();\\n  return data;\\n}",
     "rubric": "应指出缺少错误处理（网络异常、非 JSON 响应、HTTP 错误状态码）",
     "assertions": [
-      { "type": "contains", "value": "error handling", "weight": 1 },
-      { "type": "regex", "pattern": "try[\\\\s\\\\S]*catch|exception|error", "flags": "i", "weight": 1 },
+      { "type": "regex", "pattern": "try[\\\\s\\\\S]*catch|catch|exception|error", "flags": "i", "weight": 1 },
       { "type": "contains", "value": "status", "weight": 0.5 }
     ],
     "dimensions": {
@@ -154,9 +157,9 @@ export default class Init extends BaseCommand {
             console.log(tCli('cli.init.scaffolded', lang, { dir: targetDir }));
             console.log('');
             console.log(tCli('cli.init.next_steps_title', lang));
-            console.log(tCli('cli.init.next_step_edit_samples', lang));
-            console.log(tCli('cli.init.next_step_edit_skills', lang));
             console.log(tCli('cli.init.next_step_run', lang));
+            console.log(tCli('cli.init.next_step_executor', lang));
+            console.log(tCli('cli.init.next_step_customize', lang));
             console.log(tCli('cli.init.note_codex_executor', lang));
         });
     }

package/dist/cli/lib/cmd-flags.d.ts

@@ -96,7 +96,6 @@ export interface EvalFlags {
     'no-strict-baseline'?: boolean;
     effort?: string;
     'no-diagnostic'?: boolean;
-    blind?: boolean;
     repeat?: string;
     'judge-repeat'?: string;
     bootstrap?: boolean;

package/dist/cli/lib/i18n-dict/init.d.ts

@@ -1,3 +1,3 @@
 import type { CliMessage } from './types.js';
-export type InitMessageKey = 'cli.init.scaffolded' | 'cli.init.next_steps_title' | 'cli.init.next_step_edit_samples' | 'cli.init.next_step_edit_skills' | 'cli.init.next_step_run' | 'cli.init.note_codex_executor';
+export type InitMessageKey = 'cli.init.scaffolded' | 'cli.init.next_steps_title' | 'cli.init.next_step_run' | 'cli.init.next_step_executor' | 'cli.init.next_step_customize' | 'cli.init.note_codex_executor';
 export declare const initDict: Record<InitMessageKey, CliMessage>;

package/dist/cli/lib/i18n-dict/init.js

@@ -1,23 +1,26 @@
 export const initDict = {
     'cli.init.scaffolded': {
-        zh: '已初始化 omk 项目: {dir}',
+        zh: '已初始化 omk 项目：{dir}',
         en: 'omk project initialized at: {dir}',
     },
     'cli.init.next_steps_title': {
-        zh: '下一步:',
+        zh: '下一步：',
         en: 'Next steps:',
     },
-    'cli.init.next_step_edit_samples': {
-        zh: '  1. 编辑 eval-samples.json，加入你要测的评测用例',
-        en: '  1. Edit eval-samples.json to add your test cases',
+    // 先让用户「无需改任何文件直接跑通」——脚手架的用例与 skill 本身可跑(已过合规校验),
+    // 跑出第一份报告是冷启动最该先发生的事;「换成你自己的」放到跑通之后。这也消除了
+    // 主 README「不用改任何文件」与旧 init「先编辑」的矛盾。
+    'cli.init.next_step_run': {
+        zh: '  1. 直接跑通（无需先改任何文件）：omk eval --control code-review-v1 --treatment code-review-v2',
+        en: '  1. Run it as-is (no edits needed): omk eval --control code-review-v1 --treatment code-review-v2',
     },
-    'cli.init.next_step_edit_skills': {
-        zh: '  2. 编辑 skills/code-review-v1/SKILL.md 和 skills/code-review-v2/SKILL.md, 为两个 skill 版本填入实际内容',
-        en: '  2. Edit skills/code-review-v1/SKILL.md and skills/code-review-v2/SKILL.md with your skill versions',
+    'cli.init.next_step_executor': {
+        zh: '     默认执行器与评委用 claude CLI，需先装好并登录；想换别的模型或离线跑（无需 API key）见文档「执行器」。',
+        en: '     The default executor and judge use the claude CLI (install and log in first); to use another model or run offline (no API key) see the Executors docs.',
     },
-    'cli.init.next_step_run': {
-        zh: '  3. 运行: omk eval --control code-review-v1 --treatment code-review-v2',
-        en: '  3. Run: omk eval --control code-review-v1 --treatment code-review-v2',
+    'cli.init.next_step_customize': {
+        zh: '  2. 跑通后，把 skills/code-review-v1/SKILL.md 和 skills/code-review-v2/SKILL.md 与 eval-samples.json 换成你自己的 skill 和用例',
+        en: '  2. Once it runs, replace skills/code-review-v1/SKILL.md and skills/code-review-v2/SKILL.md and eval-samples.json with your own skills and cases',
     },
     'cli.init.note_codex_executor': {
         zh: '\n注: omk 评测时把 SKILL.md 整文(含 frontmatter)作为 system prompt 注入——跨 executor 一致(claude / codex / openai-api / gemini 都走同一条路径,不依赖任何 executor 的 native skill auto-discovery 或 Skill 工具机制)。frontmatter 在 prompt 头部对 model 行为无显著影响。\n模板带 Claude Code 兼容的 frontmatter(name + description)是为了让同一份 directory-skill 也能 deploy 到 Claude Code:把整个目录复制到 ~/.claude/skills/code-review-v1/(整目录,不是单个 SKILL.md),Claude SDK 才能识别。这是 omk 评测之外的 bonus,一份文件双向 dogfood。',

package/dist/cli/lib/i18n-dict/run.d.ts

@@ -1,3 +1,3 @@
 import type { CliMessage } from './types.js';
-export type RunMessageKey = 'cli.progress.preflight_starting' | 'cli.progress.sample_retry' | 'cli.progress.sample_error' | 'cli.progress.sample_executing' | 'cli.progress.sample_exec_done' | 'cli.progress.output_preview' | 'cli.progress.judging' | 'cli.progress.judged' | 'cli.progress.skipped' | 'cli.progress.sample_done' | 'cli.progress.sample_failed_done' | 'cli.run.invalid_repeat' | 'cli.run.invalid_judge_repeat' | 'cli.run.no_debias_length_active' | 'cli.run.invalid_bootstrap_samples' | 'cli.run.bootstrap_samples_too_large' | 'cli.run.dry_run_no_scores' | 'cli.run.skill_section' | 'cli.run.run_section' | 'cli.run.batch_complete' | 'cli.run.batch_verdict_header' | 'cli.run.batch_child_report_missing' | 'cli.run.eval_complete' | 'cli.run.tally' | 'cli.run.report_saved' | 'cli.run.evidence_recorded' | 'cli.run.evidence_recorded_unbound' | 'cli.run.report_only_gate_skipped' | 'cli.run.report_server_running' | 'cli.run.report_server_view' | 'cli.run.report_server_stop' | 'cli.run.no_serve_in_non_tty' | 'cli.run.no_serve_view_hint' | 'cli.run.gold_load_failed' | 'cli.run.gold_load_issue' | 'cli.run.contamination_warning' | 'cli.run.skip_connectivity_warning';
+export type RunMessageKey = 'cli.progress.preflight_starting' | 'cli.progress.sample_retry' | 'cli.progress.sample_error' | 'cli.progress.sample_executing' | 'cli.progress.sample_exec_done' | 'cli.progress.output_preview' | 'cli.progress.judging' | 'cli.progress.judged' | 'cli.progress.skipped' | 'cli.progress.sample_done' | 'cli.progress.sample_failed_done' | 'cli.run.invalid_repeat' | 'cli.run.invalid_holdout_ratio' | 'cli.run.invalid_judge_repeat' | 'cli.run.no_debias_length_active' | 'cli.run.invalid_bootstrap_samples' | 'cli.run.bootstrap_samples_too_large' | 'cli.run.dry_run_no_scores' | 'cli.run.skill_section' | 'cli.run.run_section' | 'cli.run.batch_complete' | 'cli.run.batch_verdict_header' | 'cli.run.batch_child_report_missing' | 'cli.run.eval_complete' | 'cli.run.tally' | 'cli.run.report_saved' | 'cli.run.evidence_recorded' | 'cli.run.evidence_recorded_unbound' | 'cli.run.report_only_gate_skipped' | 'cli.run.report_server_running' | 'cli.run.report_server_view' | 'cli.run.report_server_stop' | 'cli.run.no_serve_in_non_tty' | 'cli.run.no_serve_view_hint' | 'cli.run.gold_load_failed' | 'cli.run.gold_load_issue' | 'cli.run.contamination_warning' | 'cli.run.skip_connectivity_warning';
 export declare const runDict: Record<RunMessageKey, CliMessage>;

package/dist/cli/lib/i18n-dict/run.js

@@ -51,9 +51,13 @@ export const runDict = {
         zh: '⚠ --judge-repeat "{value}" 无效 (期望 ≥ 1 的整数), 已按 1 次 judge 执行\n',
         en: '⚠ --judge-repeat "{value}" is invalid (expected an integer ≥ 1), falling back to 1 judge call\n',
     },
+    'cli.run.invalid_holdout_ratio': {
+        zh: '⚠ --holdout-ratio "{value}" 无效 (期望 0 到 1 之间的小数), 已忽略、不做 holdout 切分\n',
+        en: '⚠ --holdout-ratio "{value}" is invalid (expected a fraction in (0, 1)), ignored — no holdout split\n',
+    },
     'cli.run.no_debias_length_active': {
-        zh: 'ℹ --no-debias-length 已生效: judge prompt 退回 v2-cot, 与 < v0.21 报告 hash 一致。\n',
-        en: 'ℹ --no-debias-length is active: judge prompt reverts to v2-cot, matching < v0.21 report hashes.\n',
+        zh: 'ℹ --no-debias-length 已生效：judge prompt 去掉长度去偏指令（debias-off 变体），hash 与默认开启时不同。\n',
+        en: 'ℹ --no-debias-length is active: the judge prompt drops the length-debias instruction (debias-off variant); its hash differs from the default.\n',
     },
     'cli.run.invalid_bootstrap_samples': {
         zh: '⚠ --bootstrap-samples "{value}" 无效 (期望 ≥ 100 的整数), 已按 1000 执行\n',

package/dist/cli/lib/parse-run-config.d.ts

@@ -42,10 +42,12 @@ export interface RunConfig {
     lang: 'zh' | 'en' | undefined;
     mcpConfig: string | undefined;
     verbose: boolean | undefined;
-    blind?: boolean | undefined;
     retry?: number;
     resume?: string;
     layeredStats?: boolean;
+    /** --holdout-ratio R (0 < R < 1). Hold out a deterministic sample slice; report-finalize
+     *  computes train vs holdout composite (report.analysis.holdout) for the overfitting gate. */
+    holdoutRatio?: number;
     /** --judge-repeat N. Calls LLM judge N times per (sample × dimension). Default 1. */
     judgeRepeat?: number;
     /** Unified judge config. Always non-empty; 1 entry = single judge, ≥ 2 = ensemble.

package/dist/cli/lib/parse-run-config.js

@@ -94,7 +94,6 @@ export function parseRunConfig(values) {
     const verbose = values.verbose ?? false;
     const retry = Math.max(0, Number(values.retry ?? 0) || 0);
     const resume = values.resume;
-    const blind = values.blind ?? evalConfig?.blind ?? false;
     const layeredStats = values['layered-stats'] ?? false;
     // strict-baseline default true. Reconcile both flag forms with eval.yaml fallback.
     // Priority: --no-strict-baseline > --strict-baseline > eval.yaml strictBaseline > true。
@@ -144,7 +143,6 @@ export function parseRunConfig(values) {
             verbose,
             retry,
             resume,
-            blind,
             layeredStats,
             budget: evalConfig?.budget,
             strictBaseline,

package/dist/eval-core/evaluation-job.d.ts

@@ -1,5 +1,5 @@
 import type { Artifact, EvaluationErrorCategory, EvaluationJob, EvaluationRequest, EvaluationRun, JudgeConfig } from '../types/index.js';
-export declare function buildEvaluationRequest({ samplesPath, skillDir, artifacts, model, executor, noJudge, concurrency, timeoutMs, noCache, dryRun, blind, project, owner, tags, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }: {
+export declare function buildEvaluationRequest({ samplesPath, skillDir, artifacts, model, executor, noJudge, concurrency, timeoutMs, noCache, dryRun, project, owner, tags, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }: {
     samplesPath: string;
     skillDir: string;
     artifacts: Artifact[];
@@ -10,11 +10,11 @@ export declare function buildEvaluationRequest({ samplesPath, skillDir, artifact
     timeoutMs?: number;
     noCache: boolean;
     dryRun: boolean;
-    blind: boolean;
     project?: string;
     owner?: string;
     tags?: string[];
     repeat?: number;
+    holdoutRatio?: number;
     batch?: boolean;
     judgeRepeat?: number;
     judgeModels: JudgeConfig[];

package/dist/eval-core/evaluation-job.js

@@ -1,7 +1,7 @@
 function nowIso() {
     return new Date().toISOString();
 }
-export function buildEvaluationRequest({ samplesPath, skillDir, artifacts, model, executor, noJudge, concurrency, timeoutMs, noCache, dryRun, blind, project, owner, tags, repeat, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }) {
+export function buildEvaluationRequest({ samplesPath, skillDir, artifacts, model, executor, noJudge, concurrency, timeoutMs, noCache, dryRun, project, owner, tags, repeat, holdoutRatio, batch, judgeRepeat, judgeModels, bootstrap, bootstrapSamples, lengthDebias, budget, effort, }) {
     return {
         samplesPath,
         skillDir,
@@ -13,11 +13,11 @@ export function buildEvaluationRequest({ samplesPath, skillDir, artifacts, model
         timeoutMs,
         noCache,
         dryRun,
-        blind,
         project,
         owner,
         tags,
         repeat,
+        holdoutRatio,
         batch,
         judgeRepeat,
         judgeModels,

package/dist/eval-core/evaluation-reporting.d.ts

@@ -28,7 +28,6 @@ interface AggregateReportOptions {
     layeredStats?: boolean;
 }
 export declare function aggregateReport({ runId, variants, model, judgeModel, noJudge, executorName, samples, tasks, results, totalCostUSD, artifacts, request, run, job, layeredStats, }: AggregateReportOptions): Report;
-export declare function applyBlindMode(report: Report, variants: string[], blindSeed: string): void;
 export interface PersistableReport {
     id: string;
 }

package/dist/eval-core/evaluation-reporting.js

@@ -61,10 +61,14 @@ export function getCliVersion() {
     return PKG.version;
 }
 export function getGitInfo() {
+    // stdio 静默 stderr:在非 git 目录(如 omk init 出来的 demo)里 rev-parse 会打印
+    // `fatal: not a git repository` 到终端。catch 已把失败兜成 null(报告省略 git 信息),
+    // 这条 fatal 对用户是纯噪声,吞掉它。与 skill-loader 的 GIT_PROBE_STDIO 同口径。
+    const gitProbeStdio = ['ignore', 'pipe', 'ignore'];
     try {
-        const commit = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf-8' }).trim();
-        const branch = execFileSync('git', ['rev-parse', '--abbrev-ref', 'HEAD'], { encoding: 'utf-8' }).trim();
-        const dirty = execFileSync('git', ['status', '--porcelain'], { encoding: 'utf-8' }).trim().length > 0;
+        const commit = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf-8', stdio: gitProbeStdio }).trim();
+        const branch = execFileSync('git', ['rev-parse', '--abbrev-ref', 'HEAD'], { encoding: 'utf-8', stdio: gitProbeStdio }).trim();
+        const dirty = execFileSync('git', ['status', '--porcelain'], { encoding: 'utf-8', stdio: gitProbeStdio }).trim().length > 0;
         return { commit, commitShort: commit.slice(0, 7), branch, dirty };
     }
     catch {
@@ -199,8 +203,8 @@ export function aggregateReport({ runId, variants, model, judgeModel, noJudge, e
         ...(noJudge ? {} : { runtime: getExecutorRuntimeFingerprint(jc.executor, jc.model, runtimeOptions) }),
     }));
     // length-debias is on by default; the request only sets it
-    // false when the user passed --no-debias-length. The hash differs between
-    // v3-cot-length (on) and v2-cot (off) so readers can detect the divergence.
+    // false when the user passed --no-debias-length. The judgePromptHash differs between
+    // the length-debias-on and -off prompt variants so readers can detect the divergence.
     const lengthDebiasOn = request?.lengthDebias !== false;
     const debiasModeList = [];
     if (lengthDebiasOn)
@@ -271,42 +275,6 @@ export function aggregateReport({ runId, variants, model, judgeModel, noJudge, e
             }])),
     };
 }
-export function applyBlindMode(report, variants, blindSeed) {
-    const labels = variants.map((_, i) => String.fromCharCode(65 + i));
-    let seed = parseInt(hashString(blindSeed).slice(0, 8), 16) | 0;
-    const seededRandom = () => {
-        seed |= 0;
-        seed = seed + 0x6D2B79F5 | 0;
-        let value = Math.imul(seed ^ seed >>> 15, 1 | seed);
-        value ^= value + Math.imul(value ^ value >>> 7, 61 | value);
-        return ((value ^ value >>> 14) >>> 0) / 4294967296;
-    };
-    const shuffled = [...variants];
-    for (let i = shuffled.length - 1; i > 0; i--) {
-        const j = Math.floor(seededRandom() * (i + 1));
-        [shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
-    }
-    const blindMap = Object.fromEntries(shuffled.map((variant, i) => [labels[i], variant]));
-    const reverseMap = Object.fromEntries(Object.entries(blindMap).map(([label, variant]) => [variant, label]));
-    report.meta.blind = true;
-    report.meta.blindMap = blindMap;
-    report.meta.variants = labels;
-    if (report.meta.executorRuntimes) {
-        report.meta.executorRuntimes = Object.fromEntries(Object.entries(report.meta.executorRuntimes).map(([variant, runtime]) => [reverseMap[variant] ?? variant, runtime]));
-    }
-    const newSummary = {};
-    for (const [variant, stats] of Object.entries(report.summary)) {
-        newSummary[reverseMap[variant]] = stats;
-    }
-    report.summary = newSummary;
-    for (const result of report.results) {
-        const newVariants = {};
-        for (const [variant, data] of Object.entries(result.variants)) {
-            newVariants[reverseMap[variant]] = data;
-        }
-        result.variants = newVariants;
-    }
-}
 export function persistReport(report, outputDir) {
     if (!outputDir)
         return null;

package/dist/eval-core/execution-strategy.js

@@ -101,8 +101,9 @@ export function resolveExecutionStrategy(task, model, timeoutMs, verbose, effort
         verbose,
         ...(effort && { effort }),
         // pass skill-isolation declaration to executors. undefined keeps
-        // SDK default; [] = strict isolation (skills:[] + disallowedTools:['Skill']);
-        // [...] = whitelist (skills:[...] only).
+        // SDK default; [] = strict isolation (skills:[] + disallowedTools:['Skill']).
+        // non-empty allowedSkills is rejected upstream (validateEvalConfig) and by every
+        // executor — a skill whitelist could not be fully isolated, so it was removed.
         ...(task.artifact.allowedSkills !== undefined && { allowedSkills: task.artifact.allowedSkills }),
         // Sample.mocks 透传到 executor。executor(claude-sdk / claude-cli)
         // 自决定怎么落地(in-process hook vs 临时 CLAUDE_CONFIG_DIR + on-disk hook)。

package/dist/eval-core/holdout.d.ts

@@ -0,0 +1,66 @@
+/**
+ * Holdout split + train/holdout composite breakdown.
+ *
+ * Deterministic (no RNG) train/holdout partitioning of a sample set, plus the
+ * subset-composite recompute that lets `omk eval --holdout-ratio` and `omk evolve`
+ * score a variant on a withheld slice using the *same* aggregation as the headline
+ * composite (`buildVariantSummary`). Lives in eval-core so both the eval pipeline
+ * and the authoring/evolve loop depend *down* into it (authoring → eval-core is the
+ * established direction).
+ *
+ * A large train − holdout composite gap is the generalization / sample-set-overfitting
+ * signal the verdict's overfitting gate reads (`src/eval-core/verdict.ts`).
+ */
+import type { Report, HoldoutBreakdown } from '../types/index.js';
+/** A train / holdout partition of a sample set. */
+export interface HoldoutSplit {
+    trainIds: Set<string>;
+    holdoutIds: Set<string>;
+}
+/** Below this many samples on any side, a split is too small to be meaningful —
+ *  callers fall back to full-set scoring and mark the breakdown `disabled`. */
+export declare const MIN_HOLDOUT_SUBSET = 3;
+/** Pick `count` ids at an even stride across `ids` (deterministic, no RNG) so the
+ *  picked subset is representative of the ordering and stable across rounds/runs. */
+export declare function pickByStride(ids: string[], count: number): Set<string>;
+/**
+ * Deterministically split sample ids into train / holdout by `ratio` (fraction
+ * held out). Holdout members are picked at an even stride so the partition is
+ * representative of the ordering, and the split is stable across rounds and runs
+ * (no RNG). Returns null when ratio ≤ 0 or either side would drop below
+ * MIN_HOLDOUT_SUBSET — the caller then scores on the full set.
+ */
+export declare function splitHoldout(sampleIds: string[], ratio: number): HoldoutSplit | null;
+/**
+ * Mean composite over the subset of a report's results whose sample_id is in
+ * `ids`, using the same aggregation as the full-run summary
+ * (`buildVariantSummary`) so train / holdout scores stay comparable to the
+ * headline composite. Returns 0 when the subset has no scorable entries.
+ */
+export declare function subsetCompositeScore(report: Report, variantKey: string, ids: Set<string>): number;
+/**
+ * How many subset results actually produced a usable composite (> 0) for a variant.
+ * `buildVariantSummary` averages only `compositeScore > 0` entries (schema.ts), so
+ * the mean can rest on far fewer samples than the authored split size when runs
+ * flake / partial-error / budget-abort. The overfitting gate must trust THIS count,
+ * not the authored `trainCount` / `holdoutCount`, or a 1-of-3 holdout gets dressed
+ * up as a 3-sample-backed conclusion.
+ */
+export declare function subsetScorableCount(report: Report, variantKey: string, ids: Set<string>): number;
+/**
+ * Train vs holdout composite breakdown per variant for `omk eval --holdout-ratio`.
+ * Post-hoc — never perturbs the headline aggregation or bootstrap CI.
+ *
+ * The split is taken over `sampleIdOrder` — the **stable authored sample order**
+ * (the loaded `samples` file order), NOT `report.results`, whose insertion order
+ * is the concurrent-completion order and drifts run-to-run. Binding the stride pick
+ * to the authored order is what makes the holdout (and the verdict overfitting gate
+ * it feeds) deterministic and reproducible. Subset scores are then read from
+ * `report.results` by id-set membership, which is order-independent.
+ *
+ * When the split is too small on either side (< MIN_HOLDOUT_SUBSET) it returns
+ * `{ disabled: true }` with an empty `perVariant`, so the verdict overfitting gate
+ * stays inert. The testSetHash watermark (gap-spec §7.1) is attached by the caller,
+ * shared with gapReports.
+ */
+export declare function computeHoldoutBreakdown(report: Report, variantNames: string[], ratio: number, sampleIdOrder: string[]): HoldoutBreakdown;