oh-my-knowledge 0.32.0 → 0.34.0

package/dist/authoring/generator.d.ts

@@ -1,4 +1,5 @@
-import type { Sample } from '../types/index.js';
+import type { Sample, ExecutorFn } from '../types/index.js';
+import type { ObservationInboxItem } from '../types/observability.js';
 interface GenerateSamplesOptions {
     skillContent: string;
     count?: number;
@@ -29,10 +30,36 @@ export declare function generateSamples({ skillContent, count, model, executorNa
     samples: Sample[];
     costUSD: number;
 }>;
+type TraceSignalItem = Pick<ObservationInboxItem, 'skillName' | 'signalType' | 'signalSubtype' | 'severity' | 'evidence' | 'messageWindow' | 'occurrences'>;
+/**
+ * Build the generation prompt for `omk sample --from-traces`. Renders each
+ * observation-inbox signal (evidence + message window) into a section and asks
+ * the generator to synthesize regression samples that reproduce the observed
+ * production failures. The trace text feeds the *generator* only — never the
+ * judge prompt — so judge-prompt isolation is unaffected.
+ */
+export declare function buildSamplesFromTracesPrompt(items: TraceSignalItem[], count?: number): string;
+export interface GenerateSamplesFromTracesOptions {
+    items: TraceSignalItem[];
+    count?: number;
+    model?: string;
+    executorName?: string;
+    /** Injectable executor (tests). Defaults to createExecutor(executorName). */
+    executor?: ExecutorFn;
+}
+/**
+ * Generate draft eval samples from production-trace observation signals. Mirrors
+ * generateSamples' executor / retry / finalize path but feeds the trace prompt and
+ * stamps `provenance: 'production-trace'`. Output is meant to land in a review draft,
+ * not the live dataset (the CLI enforces that).
+ */
+export declare function generateSamplesFromTraces({ items, count, model, executorName, executor: injectedExecutor, }: GenerateSamplesFromTracesOptions): Promise<{
+    samples: Sample[];
+    costUSD: number;
+}>;
 export declare function sanitizeGeneratedSamples(samples: Sample[], opts?: {
     skillContent?: string;
 }): {
     stripped: string[];
 };
 export {};
-//# sourceMappingURL=generator.d.ts.map

package/dist/authoring/generator.js

@@ -461,6 +461,119 @@ async function finalizeSamples(samples, costUSD, skillContent) {
     }
     return { samples, costUSD };
 }
+const TRACE_GEN_INSTRUCTIONS = `下面给出的不是 skill，而是从生产会话 trace 中观测到的失败 / 异常信号。请为这些信号生成评测用例（eval samples），使评测能复现并守住这些失败模式——把线上真实发生过的问题沉淀成回归用例。
+
+要求：
+- 每个信号生成 1-2 条聚焦回归用例；信号若是噪声 / 证据不足 / 无法复现，跳过它，不要硬凑。
+- prompt 要还原触发该信号的场景（自然语言任务），不要直接复述证据文本。
+- 断言优先用 mock_hit / tools_called / tools_not_called / tool_input_contains 精确锚定失败步骤，再用 contains 兜底；按「原子型」配比处理（无需工作流编号步骤），除非证据明显是多步流程。
+- 不要在输出里说明判断过程，直接输出 JSON 数组。`;
+/**
+ * Build the generation prompt for `omk sample --from-traces`. Renders each
+ * observation-inbox signal (evidence + message window) into a section and asks
+ * the generator to synthesize regression samples that reproduce the observed
+ * production failures. The trace text feeds the *generator* only — never the
+ * judge prompt — so judge-prompt isolation is unaffected.
+ */
+export function buildSamplesFromTracesPrompt(items, count) {
+    const sections = items.map((it, i) => {
+        const ev = it.evidence ?? {};
+        const evLines = [
+            ev.tool && `工具: ${ev.tool}`,
+            ev.query && `查询/输入: ${ev.query}`,
+            ev.path && `路径: ${ev.path}`,
+            ev.outputSnippet && `输出片段: ${ev.outputSnippet}`,
+            ev.assistantSnippet && `助手片段: ${ev.assistantSnippet}`,
+            ev.markerToken && `标记: ${ev.markerToken}`,
+        ].filter(Boolean).join('\n') || '(无结构化证据)';
+        const win = it.messageWindow
+            ? '\n上下文消息:\n' + [...it.messageWindow.before, ...it.messageWindow.event, ...it.messageWindow.after]
+                .map((m) => `  [${m.role}] ${m.snippet}`).join('\n')
+            : '';
+        return `### 信号 ${i + 1}：${it.signalType} / ${it.signalSubtype}（严重度 ${it.severity}，出现 ${it.occurrences} 次，skill: ${it.skillName}）\n${evLines}${win}`;
+    }).join('\n\n---\n\n');
+    const countLine = typeof count === 'number'
+        ? `共生成约 ${count} 条评测用例。`
+        : '按上面的「每个信号 1-2 条」生成。';
+    return `${TRACE_GEN_INSTRUCTIONS}
+
+## 观测到的失败信号（共 ${items.length} 个）
+
+${sections}
+
+${countLine}直接输出 JSON 数组。`;
+}
+/** Build a sanitize context string from trace evidence so finalizeSamples keeps
+ *  tool-name assertions that reference tools actually seen in the traces (an empty
+ *  context would strip every tool-name fact assertion). */
+function traceSanitizeContext(items) {
+    return items.map((it) => {
+        const ev = it.evidence ?? {};
+        return [ev.tool, ev.query, ev.path, ev.outputSnippet, ev.assistantSnippet, ev.markerToken]
+            .filter(Boolean).join(' ');
+    }).join('\n');
+}
+/**
+ * Generate draft eval samples from production-trace observation signals. Mirrors
+ * generateSamples' executor / retry / finalize path but feeds the trace prompt and
+ * stamps `provenance: 'production-trace'`. Output is meant to land in a review draft,
+ * not the live dataset (the CLI enforces that).
+ */
+export async function generateSamplesFromTraces({ items, count, model = GENERATOR_DEFAULT_MODEL, executorName = 'claude', executor: injectedExecutor, }) {
+    if (items.length === 0)
+        return { samples: [], costUSD: 0 };
+    const executor = injectedExecutor ?? createExecutor(executorName);
+    const prompt = buildSamplesFromTracesPrompt(items, count);
+    const sanitizeContext = traceSanitizeContext(items);
+    const PROVENANCE = 'production-trace';
+    const MAX_ATTEMPTS = 3;
+    let lastErr = '';
+    let totalCost = 0;
+    for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
+        const attemptPrompt = attempt === 1
+            ? prompt
+            : `${prompt}\n\n上一次输出解析失败:${lastErr}\n请严格按 JSON 规范输出(字符串内部用「」全角引号),只输出数组,不要包含其他文字。`;
+        const result = await executor({ model, system: SYSTEM_PROMPT, prompt: attemptPrompt, timeoutMs: 300_000, lean: true });
+        totalCost += result.costUSD || 0;
+        if (!result.ok) {
+            lastErr = result.error || 'unknown error';
+            if (attempt === MAX_ATTEMPTS)
+                throw new Error(`trace generation failed after ${MAX_ATTEMPTS} attempts: ${lastErr}`);
+            continue;
+        }
+        let jsonStr = result.output.trim();
+        const jsonMatch = jsonStr.match(/```(?:json)?\s*([\s\S]*?)```/);
+        if (jsonMatch)
+            jsonStr = jsonMatch[1].trim();
+        let samples;
+        try {
+            samples = JSON.parse(jsonStr);
+        }
+        catch (e) {
+            lastErr = `JSON 解析失败: ${e.message}`;
+            if (attempt === MAX_ATTEMPTS)
+                throw new Error(`trace generation failed after ${MAX_ATTEMPTS} attempts (JSON invalid): ${lastErr}`);
+            process.stderr.write(`[omk sample --from-traces] 第 ${attempt} 次输出 JSON 无效,重试中...\n`);
+            continue;
+        }
+        if (!Array.isArray(samples)) {
+            lastErr = '输出不是 JSON 数组';
+            if (attempt === MAX_ATTEMPTS)
+                throw new Error(`trace generation failed after ${MAX_ATTEMPTS} attempts: ${lastErr}`);
+            continue;
+        }
+        // Empty array = the model conservatively skipped every signal (the trace prompt
+        // explicitly permits this when signals are noise / unreproducible). That's a valid
+        // 0-result, not a failure — return it so the CLI can no-op instead of erroring.
+        if (samples.length === 0)
+            return { samples: [], costUSD: totalCost };
+        // Stamp provenance before sanitize so it survives (it's a valid enum value).
+        for (const s of samples)
+            s.provenance = PROVENANCE;
+        return await finalizeSamples(samples, totalCost, sanitizeContext);
+    }
+    throw new Error('unreachable');
+}
 /**
  * Validate + sanitize LLM-generated samples at generator boundary.
  *
@@ -717,4 +830,3 @@ export function sanitizeGeneratedSamples(samples, opts = {}) {
     }
     return { stripped };
 }
-//# sourceMappingURL=generator.js.map

package/dist/authoring/sample-fixer.d.ts

@@ -48,4 +48,3 @@ export interface FixSamplesResult {
     }>;
 }
 export declare function fixSamples(options: FixSamplesOptions): Promise<FixSamplesResult>;
-//# sourceMappingURL=sample-fixer.d.ts.map

package/dist/authoring/sample-fixer.js

@@ -210,4 +210,3 @@ ${sampleSections}
         };
     }
 }
-//# sourceMappingURL=sample-fixer.js.map

package/dist/cli/commands/doctor.d.ts

@@ -16,6 +16,8 @@ export default class Doctor extends BaseCommand {
         model: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
         samples: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
         timeout: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
+        'output-dir': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
+        dimensions: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
         'static-only': import("@oclif/core/interfaces").BooleanFlag<boolean>;
         fix: import("@oclif/core/interfaces").BooleanFlag<boolean>;
         effort: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
@@ -23,4 +25,3 @@ export default class Doctor extends BaseCommand {
     run(): Promise<void>;
 }
 export declare function pruneDoctorHistory(dir: string, skillName: string, maxKeep: number): void;
-//# sourceMappingURL=doctor.d.ts.map

package/dist/cli/commands/doctor.js

@@ -117,7 +117,7 @@ export default class Doctor extends BaseCommand {
         }),
         samples: Flags.string({
             description: bilingual({
-                zh: '样本文件路径（.json/.yaml）。不传则按 target / cwd 顺序自动发现。',
+                zh: '用例文件路径（.json/.yaml）。不传则按 target / cwd 顺序自动发现。',
                 en: 'Samples file path (.json/.yaml). Auto-detects from target / cwd if omitted.',
             }),
         }),
@@ -128,6 +128,18 @@ export default class Doctor extends BaseCommand {
             }),
             parse: numberStringParser('--timeout', { min: 1 }),
         }),
+        'output-dir': Flags.string({
+            description: bilingual({
+                zh: '报告输出目录，默认 ~/.oh-my-knowledge/doctors。',
+                en: 'Report output dir, default ~/.oh-my-knowledge/doctors.',
+            }),
+        }),
+        dimensions: Flags.string({
+            description: bilingual({
+                zh: '自定义维度配置文件（YAML），追加到内置 7 维度之后。',
+                en: 'Custom dimensions config file (YAML), appended after builtin 7 dimensions.',
+            }),
+        }),
         'static-only': Flags.boolean({
             description: bilingual({
                 zh: '离线静态模式，只跑 4 条静态 rule(skill_readable / skill_metadata / dependencies_present / samples_contract_aligned），不调 LLM。',
@@ -182,6 +194,13 @@ export default class Doctor extends BaseCommand {
             }
             // 副作用 import: 注册 7 内置维度 spec + skill_health composer rule。
             await import('../../doctor/health/register.js');
+            if (flags.dimensions) {
+                const { loadAndRegisterCustomDimensions } = await import('../../doctor/health/load-custom-dimensions.js');
+                const count = loadAndRegisterCustomDimensions(resolve(flags.dimensions));
+                if (count > 0) {
+                    process.stderr.write(lang === 'zh' ? `已加载 ${count} 个自定义维度\n` : `Loaded ${count} custom dimension(s)\n`);
+                }
+            }
             const { runDoctor } = await import('../../doctor/index.js');
             const { renderDoctorReportText, renderDoctorReportJson } = await import('../../doctor/renderer.js');
             const { getRegisteredRules } = await import('../../doctor/rules.js');
@@ -233,7 +252,7 @@ export default class Doctor extends BaseCommand {
                 }
                 renderDoctorReportText(report, lang);
             }
-            persistDoctorReport(report);
+            persistDoctorReport(report, flags['output-dir'] ? resolve(flags['output-dir']) : undefined);
             if (flags.fix) {
                 const existing = report;
                 if (existing.outcome !== 'failed') {
@@ -251,8 +270,8 @@ export default class Doctor extends BaseCommand {
 // 每个 skill 最多保留多少份历史 doctor 报告(避免无界增长拖慢 studio 启动 +
 // scanDoctorReports 扫盘成本)。50 = ~每天 1 跑撑 1.5 个月 sparkline,够用。
 const DOCTOR_HISTORY_MAX_PER_SKILL = 50;
-function persistDoctorReport(report) {
-    const dir = join(homedir(), '.oh-my-knowledge', 'doctors');
+function persistDoctorReport(report, outputDir) {
+    const dir = outputDir ?? join(homedir(), '.oh-my-knowledge', 'doctors');
     mkdirSync(dir, { recursive: true });
     const safeId = report.id.replace(/[/\\:*?"<>|]/g, '_');
     for (const skill of report.skills) {
@@ -281,7 +300,7 @@ export function pruneDoctorHistory(dir, skillName, maxKeep) {
             continue;
         try {
             const data = JSON.parse(readFileSync(join(dir, file), 'utf-8'));
-            if (data?.kind !== 'doctor' || !Array.isArray(data.skills) || data.skills.length !== 1)
+            if (data?.reportKind !== 'doctor' || !Array.isArray(data.skills) || data.skills.length !== 1)
                 continue;
             if (data.skills[0].skillName !== skillName)
                 continue;
@@ -299,4 +318,3 @@ export function pruneDoctorHistory(dir, skillName, maxKeep) {
         catch { /* ignore */ }
     }
 }
-//# sourceMappingURL=doctor.js.map

package/dist/cli/commands/eval/gold/compare.d.ts

@@ -14,4 +14,3 @@ export default class EvalGoldCompare extends BaseCommand {
     };
     run(): Promise<void>;
 }
-//# sourceMappingURL=compare.d.ts.map

package/dist/cli/commands/eval/gold/compare.js

@@ -88,4 +88,3 @@ export default class EvalGoldCompare extends BaseCommand {
         });
     }
 }
-//# sourceMappingURL=compare.js.map

package/dist/cli/commands/eval/gold/index.d.ts

@@ -6,4 +6,3 @@ export default class EvalGold extends BaseCommand {
     };
     run(): Promise<void>;
 }
-//# sourceMappingURL=index.d.ts.map

package/dist/cli/commands/eval/gold/index.js

@@ -31,4 +31,3 @@ export default class EvalGold extends BaseCommand {
         this.exit(1);
     }
 }
-//# sourceMappingURL=index.js.map

package/dist/cli/commands/eval/gold/init.d.ts

@@ -8,4 +8,3 @@ export default class EvalGoldInit extends BaseCommand {
     };
     run(): Promise<void>;
 }
-//# sourceMappingURL=init.d.ts.map

package/dist/cli/commands/eval/gold/init.js

@@ -48,4 +48,3 @@ export default class EvalGoldInit extends BaseCommand {
         });
     }
 }
-//# sourceMappingURL=init.js.map

package/dist/cli/commands/eval/gold/validate.d.ts

@@ -9,4 +9,3 @@ export default class EvalGoldValidate extends BaseCommand {
     };
     run(): Promise<void>;
 }
-//# sourceMappingURL=validate.d.ts.map

package/dist/cli/commands/eval/gold/validate.js

@@ -43,4 +43,3 @@ export default class EvalGoldValidate extends BaseCommand {
         });
     }
 }
-//# sourceMappingURL=validate.js.map

package/dist/cli/commands/eval/index.d.ts

@@ -9,6 +9,8 @@ export default class Eval extends BaseCommand {
         lang: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
         control: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
         treatment: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
+        'control-cwd': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
+        'treatment-cwd': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
         config: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
         samples: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
         'skill-dir': import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
@@ -51,4 +53,3 @@ export default class Eval extends BaseCommand {
     };
     run(): Promise<void>;
 }
-//# sourceMappingURL=index.d.ts.map

package/dist/cli/commands/eval/index.js

@@ -7,6 +7,7 @@ import { tCli } from '../../lib/i18n.js';
 import { parseRunConfig } from '../../lib/parse-run-config.js';
 import { makeOnProgress } from '../../lib/progress.js';
 import { computeRunTally } from '../../lib/run-tally.js';
+import { DEFAULT_BOOTSTRAP_SAMPLES } from '../../../eval-core/bootstrap.js';
 function isDryRunReport(report) {
     return Boolean(report && typeof report === 'object' && report.dryRun === true);
 }
@@ -44,7 +45,7 @@ async function emitEvaluationVerdict(report, values) {
 }
 function batchItemFallbackReport(batch, item) {
     return {
-        kind: 'evaluation',
+        reportKind: 'evaluation',
         id: item.reportId,
         meta: {
             ...batch.meta,
@@ -64,7 +65,7 @@ async function loadBatchChildReports(batch, reportsDir, lang) {
     const reports = [];
     for (const item of batch.items) {
         const loaded = await store.get(item.reportId);
-        if (loaded?.kind === 'evaluation') {
+        if (loaded?.reportKind === 'evaluation') {
             reports.push(loaded);
         }
         else {
@@ -99,7 +100,7 @@ async function emitBatchVerdict(report, reportsDir, values, lang) {
 }
 async function announceSavedReport({ report, filePath, reportsDir, values, lang, }) {
     const tally = computeRunTally(report);
-    process.stderr.write(tCli(report.kind === 'batch-evaluation' ? 'cli.run.batch_complete' : 'cli.run.eval_complete', lang));
+    process.stderr.write(tCli(report.reportKind === 'batch-evaluation' ? 'cli.run.batch_complete' : 'cli.run.eval_complete', lang));
     process.stderr.write(tCli('cli.run.tally', lang, tally));
     process.stderr.write(tCli('cli.run.report_saved', lang, { path: filePath }));
     if (!values['no-serve'] && process.stdout.isTTY) {
@@ -162,11 +163,11 @@ async function runEval(_args, flags, lang) {
     if (bootstrapEnabled) {
         config.bootstrap = true;
         const bsRaw = values['bootstrap-samples'];
-        const parsedBs = bsRaw !== undefined ? Number(bsRaw) : (evalConfig?.bootstrapSamples ?? 1000);
+        const parsedBs = bsRaw !== undefined ? Number(bsRaw) : (evalConfig?.bootstrapSamples ?? DEFAULT_BOOTSTRAP_SAMPLES);
         if (bsRaw !== undefined && (!Number.isFinite(parsedBs) || parsedBs < 100)) {
             process.stderr.write(tCli('cli.run.invalid_bootstrap_samples', lang, { value: bsRaw }));
         }
-        const bsCount = Math.max(100, Math.floor(parsedBs) || 1000);
+        const bsCount = Math.max(100, Math.floor(parsedBs) || DEFAULT_BOOTSTRAP_SAMPLES);
         if (bsCount > 10000) {
             process.stderr.write(tCli('cli.run.bootstrap_samples_too_large', lang, { n: bsCount }));
         }
@@ -289,12 +290,24 @@ export default class Eval extends BaseCommand {
         lang: LANG_FLAG,
         // ── 实验角色 ──
         control: Flags.string({
-            description: bilingual({ zh: 'control variant 表达式', en: 'Control variant expr' }),
+            description: bilingual({ zh: 'control variant 表达式（仅 artifact 身份）', en: 'Control variant expr (artifact identity only)' }),
         }),
         treatment: Flags.string({
             description: bilingual({
-                zh: 'treatment variant 列表，逗号分隔',
-                en: 'Treatment variants, comma-separated',
+                zh: 'treatment variant 列表，逗号分隔（仅 artifact 身份）',
+                en: 'Treatment variants, comma-separated (artifact identity only)',
+            }),
+        }),
+        'control-cwd': Flags.string({
+            description: bilingual({
+                zh: 'control 的 runtime context 目录',
+                en: 'Runtime context dir for control',
+            }),
+        }),
+        'treatment-cwd': Flags.string({
+            description: bilingual({
+                zh: 'treatment 的 runtime context 目录列表，逗号分隔、与 --treatment 按序对齐（空位 = 无 cwd）',
+                en: 'Runtime context dirs for treatments, comma-separated, index-aligned with --treatment (blank = none)',
             }),
         }),
         config: Flags.string({
@@ -302,7 +315,7 @@ export default class Eval extends BaseCommand {
         }),
         samples: Flags.string({
             description: bilingual({
-                zh: '样本文件路径。默认 eval-samples.json，也接受 .yaml/.yml；自动发现 --skill-dir 下的 <skill>/.omk/samples.json。',
+                zh: '用例文件路径。默认 eval-samples.json，也接受 .yaml/.yml；自动发现 --skill-dir 下的 <skill>/.omk/samples.json。',
                 en: 'Samples file path. Defaults to eval-samples.json (also .yaml/.yml); auto-discovers <skill>/.omk/samples.json under --skill-dir.',
             }),
         }),
@@ -343,7 +356,7 @@ export default class Eval extends BaseCommand {
             parse: integerStringParser('--concurrency', { min: 1 }),
         }),
         timeout: Flags.string({
-            description: bilingual({ zh: '单样本超时秒，默认 600', en: 'Per-sample timeout sec, default 600' }),
+            description: bilingual({ zh: '单用例超时秒，默认 600', en: 'Per-sample timeout sec, default 600' }),
             parse: numberStringParser('--timeout', { min: 1 }),
         }),
         batch: Flags.boolean({
@@ -462,4 +475,3 @@ export default class Eval extends BaseCommand {
         });
     }
 }
-//# sourceMappingURL=index.js.map

package/dist/cli/commands/evolve.d.ts

@@ -31,7 +31,13 @@ export default class Evolve extends BaseCommand {
         'sample-fix-max-attempts': import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
         'reuse-latest-eval': import("@oclif/core/interfaces").BooleanFlag<boolean>;
         'improve-mode': import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
+        'holdout-ratio': import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
+        'no-significance-gate': import("@oclif/core/interfaces").BooleanFlag<boolean>;
+        'significance-alpha': import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
+        'test-ratio': import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
+        'edit-budget': import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
+        'no-edit-budget': import("@oclif/core/interfaces").BooleanFlag<boolean>;
+        'no-reject-memory': import("@oclif/core/interfaces").BooleanFlag<boolean>;
     };
     run(): Promise<void>;
 }
-//# sourceMappingURL=evolve.d.ts.map

package/dist/cli/commands/evolve.js

@@ -26,6 +26,13 @@ export async function runEvolve(args, flags, lang) {
         console.error(tCli('cli.evolve.specify_skill_path', lang));
         throw new CliExit(1);
     }
+    // A locked test set needs a separate val set to decide on; --test-ratio alone is a no-op.
+    if ((Number(flags['test-ratio']) || 0) > 0 && (Number(flags['holdout-ratio']) || 0) === 0) {
+        console.error(lang === 'zh'
+            ? '--test-ratio 需要配合 --holdout-ratio 使用（test 集只在留出 val 集时才有意义）。'
+            : '--test-ratio requires --holdout-ratio (a locked test set only makes sense alongside a held-out val set).');
+        throw new CliExit(2);
+    }
     const { resolveSkillInput } = await import('../lib/resolve-skill-input.js');
     let resolvedInput;
     try {
@@ -68,9 +75,17 @@ export async function runEvolve(args, flags, lang) {
             autoFixSamples: flags['auto-fix-samples'],
             sampleFixMaxAttempts: Math.max(1, Number(flags['sample-fix-max-attempts']) || 2),
             reuseLatestEval: flags['reuse-latest-eval'],
+            holdoutRatio: Number(flags['holdout-ratio']) || 0,
+            significanceGate: !flags['no-significance-gate'],
+            // parser 已保证是合法数字串；用 Number() 直取，不用 `|| default`——否则
+            // `--edit-budget 0`（关预算）/ `--significance-alpha 0` 会被 falsy 吞成默认值。
+            significanceAlpha: Number(flags['significance-alpha']),
+            testRatio: Number(flags['test-ratio']),
+            editBudget: flags['no-edit-budget'] ? 0 : Number(flags['edit-budget']),
+            rejectMemory: !flags['no-reject-memory'],
             improveMode: flags['improve-mode'] === 'rewrite' ? 'rewrite' : 'agent',
             onProgress: makeOnProgress(lang),
-            onRoundProgress({ round, totalRounds: _totalRounds, phase, score, delta, accepted, costUSD, costReported, error }) {
+            onRoundProgress({ round, totalRounds: _totalRounds, phase, score, delta, accepted, costUSD, costReported, error, significant }) {
                 // costReported=false 时显示「—」而不是 $0.0000(executor 不报 cost,如 codex)。
                 const fmtRoundCost = (c, r) => r ? `$${c.toFixed(4)}` : '—';
                 if (phase === 'baseline') {
@@ -85,7 +100,9 @@ export async function runEvolve(args, flags, lang) {
                 }
                 else if (phase === 'done') {
                     const delta_ = delta >= 0 ? `+${delta.toFixed(2)}` : delta.toFixed(2);
-                    const status = accepted ? '✓ ACCEPT' : '✗ REJECT';
+                    // 门拒掉「均分上升但不显著」的候选时，补一句原因，否则 (+0.0x) ✗ REJECT 看着矛盾。
+                    const rejectNote = !accepted && significant === false ? tCli('cli.evolve.reject_not_significant', lang) : '';
+                    const status = accepted ? '✓ ACCEPT' : `✗ REJECT${rejectNote}`;
                     process.stderr.write(tCli('cli.evolve.round_done', lang, {
                         round, score: score.toFixed(2), delta: delta_, status, cost: fmtRoundCost(costUSD, costReported !== false),
                     }));
@@ -102,6 +119,24 @@ export async function runEvolve(args, flags, lang) {
             start: result.startScore.toFixed(2), final: result.finalScore.toFixed(2),
             percent: improvement, rounds: result.totalRounds, cost: totalCostStr,
         }));
+        if (result.holdout) {
+            process.stderr.write(result.holdout.disabled
+                ? tCli('cli.evolve.holdout_disabled', lang, { ratio: result.holdout.ratio })
+                : tCli('cli.evolve.holdout_active', lang, {
+                    train: result.holdout.trainCount, holdout: result.holdout.holdoutCount,
+                }));
+        }
+        if (result.gate?.underpowered) {
+            process.stderr.write(tCli('cli.evolve.gate_underpowered', lang));
+        }
+        if (result.test?.disabled) {
+            process.stderr.write(tCli('cli.evolve.test_disabled', lang, { ratio: result.test.ratio }));
+        }
+        else if (result.test && typeof result.generalizationScore === 'number') {
+            process.stderr.write(tCli('cli.evolve.generalization', lang, {
+                count: result.test.count, score: result.generalizationScore.toFixed(2),
+            }));
+        }
         process.stderr.write(tCli('cli.evolve.best_path', lang, {
             best: result.bestSkillPath, target: resolve(skillPath),
         }));
@@ -166,7 +201,7 @@ export default class Evolve extends BaseCommand {
         }),
         samples: Flags.string({
             description: bilingual({
-                zh: '样本文件路径，默认 eval-samples.json',
+                zh: '用例文件路径，默认 eval-samples.json',
                 en: 'Samples file, default eval-samples.json',
             }),
             default: 'eval-samples.json',
@@ -198,7 +233,7 @@ export default class Evolve extends BaseCommand {
             parse: integerStringParser('--concurrency', { min: 1 }),
         }),
         timeout: Flags.string({
-            description: bilingual({ zh: '单样本超时秒，默认 600', en: 'Per-sample timeout sec, default 600' }),
+            description: bilingual({ zh: '单用例超时秒，默认 600', en: 'Per-sample timeout sec, default 600' }),
             default: '600',
             parse: numberStringParser('--timeout', { min: 1 }),
         }),
@@ -236,7 +271,7 @@ export default class Evolve extends BaseCommand {
         }),
         'stop-on-assertions-pass': Flags.boolean({
             description: bilingual({
-                zh: '普通样本断言全过时提前停止',
+                zh: '普通用例断言全过时提前停止',
                 en: 'Stop early when normal samples pass assertions',
             }),
             default: false,
@@ -271,6 +306,59 @@ export default class Evolve extends BaseCommand {
             default: 'agent',
             options: ['agent', 'rewrite'],
         }),
+        'holdout-ratio': Flags.string({
+            description: bilingual({
+                zh: '留出验收集比例（0..1，默认 0=关）。> 0 时按 holdout 分接受候选、weak-sample 只取训练集，防 train-on-test',
+                en: 'Holdout fraction for the accept decision (0..1, default 0=off). When > 0, candidates are accepted on holdout score and weak samples come only from train — guards against train-on-test',
+            }),
+            default: '0',
+            parse: numberStringParser('--holdout-ratio', { min: 0, max: 1 }),
+        }),
+        'no-significance-gate': Flags.boolean({
+            description: bilingual({
+                zh: '关掉显著性接受门，退回「候选分高一点点就收」的点估计判定（默认门开：只收统计显著的提升）',
+                en: 'Disable the significance accept gate, reverting to point-estimate accept (default: gate on — accept only statistically significant gains)',
+            }),
+            default: false,
+        }),
+        'significance-alpha': Flags.string({
+            description: bilingual({
+                zh: '显著性门的 diff CI 显著性水平（默认 0.05 = 95% CI）',
+                en: 'Significance level for the accept gate diff CI (default 0.05 = 95% CI)',
+            }),
+            default: '0.05',
+            parse: numberStringParser('--significance-alpha', { min: 0, max: 1 }),
+        }),
+        'test-ratio': Flags.string({
+            description: bilingual({
+                zh: '锁定 test 集比例（0..1，默认 0=关），需配 --holdout-ratio。全程不参与选择，收尾读一次给无偏泛化分',
+                en: 'Locked test fraction (0..1, default 0=off); requires --holdout-ratio. Never used for selection; read once at the end for an unbiased generalization score',
+            }),
+            default: '0',
+            parse: numberStringParser('--test-ratio', { min: 0, max: 1 }),
+        }),
+        'edit-budget': Flags.string({
+            description: bilingual({
+                zh: '单轮最多改动的 skill 行占比（默认 0.2）。超预算的候选评测前直接判拒，省 eval 成本',
+                en: 'Max fraction of skill lines a round may change (default 0.2). Over-budget candidates are rejected before evaluation, saving eval cost',
+            }),
+            default: '0.2',
+            parse: numberStringParser('--edit-budget', { min: 0, max: 1 }),
+        }),
+        'no-edit-budget': Flags.boolean({
+            description: bilingual({
+                zh: '关掉 edit budget 约束（允许任意大小的单轮改动）',
+                en: 'Disable the edit budget (allow arbitrarily large single-round edits)',
+            }),
+            default: false,
+        }),
+        'no-reject-memory': Flags.boolean({
+            description: bilingual({
+                zh: '关掉 rejected-edit 记忆（不把被拒改法回灌下一轮 prompt）',
+                en: 'Disable rejected-edit memory (do not feed rejected edits back into the next prompt)',
+            }),
+            default: false,
+        }),
     };
     async run() {
         const { args, flags } = await this.parse(Evolve);
@@ -280,4 +368,3 @@ export default class Evolve extends BaseCommand {
         });
     }
 }
-//# sourceMappingURL=evolve.js.map

package/dist/cli/commands/init.d.ts

@@ -13,4 +13,3 @@ export default class Init extends BaseCommand {
     };
     run(): Promise<void>;
 }
-//# sourceMappingURL=init.d.ts.map

package/dist/cli/commands/init.js

@@ -149,4 +149,3 @@ export default class Init extends BaseCommand {
         });
     }
 }
-//# sourceMappingURL=init.js.map

package/dist/cli/commands/install.d.ts

@@ -0,0 +1,22 @@
+import { BaseCommand } from '../oclif/base-command.js';
+export default class Install extends BaseCommand {
+    static description: string;
+    static examples: {
+        description: string;
+        command: string;
+    }[];
+    static args: {
+        input: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
+    };
+    static flags: {
+        lang: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
+        kind: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
+        to: import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
+        dest: import("@oclif/core/interfaces").OptionFlag<string | undefined, import("@oclif/core/interfaces").CustomOptions>;
+        force: import("@oclif/core/interfaces").BooleanFlag<boolean>;
+        'dry-run': import("@oclif/core/interfaces").BooleanFlag<boolean>;
+    };
+    run(): Promise<void>;
+    private installBuiltinAgentSkill;
+    private installManagedSkill;
+}