oh-my-knowledge 0.32.0 → 0.34.0

package/dist/authoring/evolver.d.ts

@@ -15,9 +15,96 @@ interface WeakSample {
         none?: string;
     };
 }
-export declare function extractWeakSamples(report: Report, variantKey: string, count?: number): WeakSample[];
+export declare function extractWeakSamples(report: Report, variantKey: string, count?: number, sampleIdFilter?: Set<string>): WeakSample[];
+/** A train / holdout partition of a sample set. */
+interface HoldoutSplit {
+    trainIds: Set<string>;
+    holdoutIds: Set<string>;
+}
+/** A train / val / test partition. `val` drives the accept decision; `test` is
+ *  locked — never seen during the loop, read once at the end for an unbiased
+ *  generalization score. */
+interface TrainValTestSplit {
+    trainIds: Set<string>;
+    valIds: Set<string>;
+    testIds: Set<string>;
+}
+/** Below this many decision (val) samples the bootstrap diff CI almost never
+ *  excludes 0 for realistic effect sizes, so the significance gate would reject
+ *  every candidate. Under that floor evolve degrades to the point-estimate accept
+ *  and flags `gate.underpowered`. */
+export declare const MIN_GATE_SAMPLES = 8;
+/**
+ * Deterministically split sample ids into train / holdout by `ratio` (fraction
+ * held out). Holdout members are picked at an even stride so the partition is
+ * representative of the ordering, and the split is stable across rounds and runs
+ * (no RNG). Returns null when ratio ≤ 0 or either side would drop below
+ * MIN_HOLDOUT_SUBSET — the caller then scores on the full set.
+ */
+export declare function splitHoldout(sampleIds: string[], ratio: number): HoldoutSplit | null;
+/**
+ * Deterministically split sample ids into train / val / test. `val` is carved
+ * first at an even stride; `test` is carved at an even stride over what remains,
+ * so the three sets are disjoint and stable across rounds/runs (no RNG). Returns
+ * null when either ratio ≤ 0 or any of the three sides would drop below
+ * MIN_HOLDOUT_SUBSET — the caller then degrades to a 2-way (or full-set) split.
+ */
+export declare function splitTrainValTest(sampleIds: string[], valRatio: number, testRatio: number): TrainValTestSplit | null;
+export interface AcceptDecision {
+    accepted: boolean;
+    /** Diff CI (candidate − best) when the gate ran; absent when it degraded. */
+    diffCI?: {
+        low: number;
+        high: number;
+        estimate: number;
+        significant: boolean;
+    };
+    /** True when the gate was requested but the decision set was below MIN_GATE_SAMPLES,
+     *  so the decision degraded to the point-estimate comparison. */
+    underpowered: boolean;
+}
+/**
+ * The accept decision for one round. With the significance gate on and enough
+ * decision samples, a candidate is accepted only when it is **significantly** above
+ * the current best's fresh re-eval (`bootstrapDiffCI(...).significant && estimate > 0`)
+ * AND its decision score actually beats the recorded best (`pointCand > pointBest`).
+ * The second clause preserves evolve's monotonic invariant: `bestScore` never
+ * decreases. Without it, an unlucky (noise-low) re-eval of the current best could let
+ * a candidate that is significantly above that re-eval — yet still below the recorded
+ * best — win and overwrite the best downward. Off, or under-powered, the gate degrades
+ * to the legacy point-estimate comparison alone. Pure (modulo the seeded bootstrap) so
+ * the core behavior is unit-testable.
+ */
+export declare function decideAccept(bestScores: number[], candScores: number[], pointBest: number, pointCand: number, opts: {
+    significanceGate: boolean;
+    alpha: number;
+    seed: number;
+}): AcceptDecision;
+/**
+ * A view of `report` whose results are restricted to `sampleIds`. Used to keep the
+ * holdout split out of the sample-fixer: under an active holdout, only training-split
+ * samples may enter the --auto-fix-samples prompt or be rewritten — otherwise the
+ * skill's samples get tuned to the very samples that decide acceptance, reintroducing
+ * the leak holdout exists to prevent.
+ */
+export declare function restrictReportToSamples(report: Report, sampleIds: Set<string>): Report;
 export declare function allNonTripwireAssertionsPass(report: Report, variantKey: string): boolean;
-export declare function buildImprovementPrompt(skillContent: string, score: number, weakSamples: WeakSample[]): string;
+interface EditDelta {
+    /** Symmetric line difference (added + removed unique lines) over original line count. */
+    ratio: number;
+    /** Absolute count of added + removed unique lines. */
+    changedLines: number;
+    /** Compact `+`/`-` summary of the changed lines, truncated. */
+    summary: string;
+}
+/**
+ * How a candidate differs from the current best, by trimmed non-empty line sets.
+ * `ratio` drives the edit budget; `summary` feeds the rejected-edit memory so the
+ * improver doesn't re-propose changes that already failed. Order-insensitive and
+ * O(n) over small skill files.
+ */
+export declare function computeEditDelta(before: string, after: string, maxSummaryLines?: number): EditDelta;
+export declare function buildImprovementPrompt(skillContent: string, score: number, weakSamples: WeakSample[], rejectedEdits?: string[]): string;
 /** @deprecated Use ProgressCallback from evaluation-core.ts */
 export type EvolveProgressInfo = Parameters<ProgressCallback>[0];
 export interface EvolveRoundProgressInfo {
@@ -33,6 +120,10 @@ export interface EvolveRoundProgressInfo {
     costReported?: boolean;
     error?: string;
     reused?: boolean;
+    /** When the significance gate ran: whether the candidate's gain was significant.
+     *  False on a rejected round means "score rose but within noise" — lets the CLI
+     *  explain an otherwise-confusing `(+0.0x) ✗ REJECT`. Undefined = gate didn't run. */
+    significant?: boolean;
 }
 interface EvolveOptions {
     skillPath: string;
@@ -61,15 +152,62 @@ interface EvolveOptions {
     noDiagnostic?: boolean;
     /** 跳过 doctor 健康检查门禁。默认 false。 */
     skipDoctor?: boolean;
+    /** Fraction of samples held out for the accept decision (0..1). Default 0 = off.
+     *  When > 0, a candidate is accepted on its **holdout** composite rather than the
+     *  training composite, and weak-sample extraction only sees the training split —
+     *  so the skill is never tuned to the samples that judge it. Too small a split
+     *  (either side < MIN_HOLDOUT_SUBSET) falls back to full-set scoring + a warning. */
+    holdoutRatio?: number;
+    /** Statistically gate acceptance: a candidate is accepted only when its
+     *  per-sample composite is **significantly** above the current best on the
+     *  decision (val) set — `bootstrapDiffCI(...).significant && estimate > 0` —
+     *  not merely numerically higher. Default true (rejecting improvements
+     *  indistinguishable from judge noise is the point). Below MIN_GATE_SAMPLES
+     *  decision samples the gate is underpowered and degrades to the point-estimate
+     *  comparison + a warning. Set false to force the legacy point-estimate accept. */
+    significanceGate?: boolean;
+    /** Significance level for the accept gate's diff CI. Default 0.05 (95% CI). */
+    significanceAlpha?: number;
+    /** Fraction of samples locked away as a **test** set (0..1). Default 0 = off.
+     *  Only honored alongside `holdoutRatio` > 0 (test needs a separate val set to
+     *  decide on). The test split is never seen during the loop — not by weak-sample
+     *  extraction, not by the accept gate — and is read exactly once at the end for an
+     *  unbiased `generalizationScore`. Too small a 3-way split degrades to 2-way. */
+    testRatio?: number;
+    /** Max fraction of skill lines a single round may change before the candidate is
+     *  rejected **without paying for evaluation**. Default 0.2 (matches the "≤20%"
+     *  the improvement prompt already asks for — this enforces it). A small floor
+     *  always permits a handful of lines so tiny skills aren't frozen. Set 0 to disable. */
+    editBudget?: number;
+    /** Feed rejected candidate edits back into the next round's improvement prompt
+     *  ("these were tried and did not help — don't repeat them"). Default true. */
+    rejectMemory?: boolean;
     onProgress?: ProgressCallback | null;
     onRoundProgress?: ((progress: EvolveRoundProgressInfo) => void) | null;
 }
 interface TrajectoryEntry {
     round: number;
+    /** Accept-decision score: val composite when a holdout split is active, else full-set. */
     score: number;
     delta: number;
     accepted: boolean;
     costUSD: number;
+    /** Present when a holdout split is active: the training-split composite (improvement signal). */
+    trainScore?: number;
+    /** Present when a holdout split is active: the val-split composite (== score). */
+    holdoutScore?: number;
+    /** Significance-gate diff CI (candidate − current best) on the decision set, when
+     *  the gate was powered enough to run. `significant` 决定接受。 */
+    diffCI?: {
+        low: number;
+        high: number;
+        estimate: number;
+        significant: boolean;
+    };
+    /** Fraction of skill lines this candidate changed vs the current best. */
+    editRatio?: number;
+    /** True when the candidate was rejected by the edit budget before evaluation. */
+    rejectedPreEval?: boolean;
 }
 export interface EvolveResult {
     startScore: number;
@@ -86,6 +224,34 @@ export interface EvolveResult {
     reusedBaselineReportId?: string;
     /** False = 任一轮的 exec / judge 不报 cost → totalCostUSD 是 lower-bound 而非真值。 */
     costReported?: boolean;
+    /** Holdout split summary when `--holdout-ratio` > 0. `disabled` is true when the
+     *  split was too small and evolve fell back to full-set scoring (CLI formats the
+     *  user-facing message bilingually). */
+    holdout?: {
+        ratio: number;
+        trainCount: number;
+        holdoutCount: number;
+        disabled?: boolean;
+    };
+    /** Locked-test split summary. `disabled` is true when `--test-ratio` was requested
+     *  but the 3-way split was too small, so evolve fell back to a 2-way holdout and
+     *  produced no generalization score. */
+    test?: {
+        ratio: number;
+        count: number;
+        disabled?: boolean;
+    };
+    /** Unbiased composite of the best skill on the locked test set — the headline honest
+     *  number. Present only when a 3-way split was active. The test set never influenced
+     *  selection or weak-sample extraction, so this is an out-of-sample estimate. */
+    generalizationScore?: number;
+    /** Accept-gate summary. `underpowered` = the decision set was below MIN_GATE_SAMPLES
+     *  at least once, so the gate degraded to the point-estimate comparison + warned. */
+    gate?: {
+        enabled: boolean;
+        alpha: number;
+        underpowered?: boolean;
+    };
     trajectory: TrajectoryEntry[];
     bestSkillPath: string;
     allVersions: string[];
@@ -97,6 +263,5 @@ export interface RoundReport {
     report: Report;
 }
 export declare function mergeEvolveReports(roundReports: RoundReport[], skillName: string, totalCostUSD: number, samples?: Sample[], skillPath?: string): Report;
-export declare function evolveSkill({ skillPath, samplesPath, rounds, target, stopOnAssertionsPass, autoFixSamples, sampleFixMaxAttempts, reuseLatestEval, model, judgeModels, improveModel, improveMode, executorName, concurrency, timeoutMs, skipConnectivity, effort, noDiagnostic, skipDoctor, onProgress, onRoundProgress, }: EvolveOptions): Promise<EvolveResult>;
+export declare function evolveSkill({ skillPath, samplesPath, rounds, target, stopOnAssertionsPass, autoFixSamples, sampleFixMaxAttempts, reuseLatestEval, model, judgeModels, improveModel, improveMode, executorName, concurrency, timeoutMs, skipConnectivity, effort, noDiagnostic, skipDoctor, holdoutRatio, significanceGate, significanceAlpha, testRatio, editBudget, rejectMemory, onProgress, onRoundProgress, }: EvolveOptions): Promise<EvolveResult>;
 export {};
-//# sourceMappingURL=evolver.d.ts.map

package/dist/authoring/evolver.js

@@ -6,6 +6,8 @@ import { persistReport, DEFAULT_OUTPUT_DIR, generateRunId, hashString } from '..
 import { createFileStore } from '../server/report-store.js';
 import { analyzeResults } from '../analysis/report-diagnostics.js';
 import { loadSamples } from '../inputs/load-samples.js';
+import { buildVariantSummary } from '../eval-core/schema.js';
+import { bootstrapDiffCI, DEFAULT_BOOTSTRAP_ALPHA, DEFAULT_BOOTSTRAP_SAMPLES } from '../eval-core/bootstrap.js';
 import { fixSamples } from './sample-fixer.js';
 const IMPROVE_SYSTEM_PROMPT = `你是一个 AI 提示词改进专家。你的任务是分析评测结果中的薄弱环节，针对性地改进 skill（系统提示词），使其在评测中获得更高的分数。
 
@@ -121,9 +123,11 @@ function readSkillName(skillPath) {
         return null;
     }
 }
-export function extractWeakSamples(report, variantKey, count = 5) {
+export function extractWeakSamples(report, variantKey, count = 5, sampleIdFilter) {
     const weakSamples = [];
     for (const r of report.results) {
+        if (sampleIdFilter && !sampleIdFilter.has(r.sample_id))
+            continue;
         const v = r.variants[variantKey];
         if (!v || typeof v.compositeScore !== 'number')
             continue;
@@ -153,6 +157,134 @@ export function extractWeakSamples(report, variantKey, count = 5) {
         .sort((a, b) => a.compositeScore - b.compositeScore)
         .slice(0, count);
 }
+/** Below this many samples on any side, a split is too small to be meaningful —
+ *  evolve falls back to full-set scoring and warns. */
+const MIN_HOLDOUT_SUBSET = 3;
+/** Below this many decision (val) samples the bootstrap diff CI almost never
+ *  excludes 0 for realistic effect sizes, so the significance gate would reject
+ *  every candidate. Under that floor evolve degrades to the point-estimate accept
+ *  and flags `gate.underpowered`. */
+export const MIN_GATE_SAMPLES = 8;
+/** Pick `count` ids at an even stride across `ids` (deterministic, no RNG) so the
+ *  picked subset is representative of the ordering and stable across rounds/runs. */
+function pickByStride(ids, count) {
+    const picked = new Set();
+    if (count <= 0)
+        return picked;
+    const stride = ids.length / count;
+    for (let k = 0; k < count; k++)
+        picked.add(ids[Math.floor(k * stride)]);
+    return picked;
+}
+/**
+ * Deterministically split sample ids into train / holdout by `ratio` (fraction
+ * held out). Holdout members are picked at an even stride so the partition is
+ * representative of the ordering, and the split is stable across rounds and runs
+ * (no RNG). Returns null when ratio ≤ 0 or either side would drop below
+ * MIN_HOLDOUT_SUBSET — the caller then scores on the full set.
+ */
+export function splitHoldout(sampleIds, ratio) {
+    if (!(ratio > 0) || sampleIds.length === 0)
+        return null;
+    const holdoutCount = Math.round(sampleIds.length * ratio);
+    const trainCount = sampleIds.length - holdoutCount;
+    if (holdoutCount < MIN_HOLDOUT_SUBSET || trainCount < MIN_HOLDOUT_SUBSET)
+        return null;
+    const holdoutIds = pickByStride(sampleIds, holdoutCount);
+    const trainIds = new Set(sampleIds.filter((id) => !holdoutIds.has(id)));
+    return { trainIds, holdoutIds };
+}
+/**
+ * Deterministically split sample ids into train / val / test. `val` is carved
+ * first at an even stride; `test` is carved at an even stride over what remains,
+ * so the three sets are disjoint and stable across rounds/runs (no RNG). Returns
+ * null when either ratio ≤ 0 or any of the three sides would drop below
+ * MIN_HOLDOUT_SUBSET — the caller then degrades to a 2-way (or full-set) split.
+ */
+export function splitTrainValTest(sampleIds, valRatio, testRatio) {
+    if (!(valRatio > 0) || !(testRatio > 0) || sampleIds.length === 0)
+        return null;
+    const valCount = Math.round(sampleIds.length * valRatio);
+    const testCount = Math.round(sampleIds.length * testRatio);
+    const trainCount = sampleIds.length - valCount - testCount;
+    if (valCount < MIN_HOLDOUT_SUBSET || testCount < MIN_HOLDOUT_SUBSET || trainCount < MIN_HOLDOUT_SUBSET)
+        return null;
+    const valIds = pickByStride(sampleIds, valCount);
+    const remaining = sampleIds.filter((id) => !valIds.has(id));
+    const testIds = pickByStride(remaining, testCount);
+    const trainIds = new Set(sampleIds.filter((id) => !valIds.has(id) && !testIds.has(id)));
+    return { trainIds, valIds, testIds };
+}
+/**
+ * Mean composite over the subset of a report's results whose sample_id is in
+ * `ids`, using the same aggregation as the full-run summary
+ * (`buildVariantSummary`) so train / holdout scores stay comparable to the
+ * headline composite. Returns 0 when the subset has no scorable entries.
+ */
+function subsetCompositeScore(report, variantKey, ids) {
+    const entries = [];
+    for (const r of report.results) {
+        if (!ids.has(r.sample_id))
+            continue;
+        const v = r.variants[variantKey];
+        if (v)
+            entries.push(v);
+    }
+    if (entries.length === 0)
+        return 0;
+    return buildVariantSummary(entries).avgCompositeScore ?? 0;
+}
+/**
+ * Per-sample composite scores over the subset of a report's results whose
+ * sample_id is in `ids`, in result order. Feeds `bootstrapDiffCI` for the
+ * significance accept gate — the array (not the mean) is what the bootstrap
+ * resamples. Entries without a numeric compositeScore are skipped.
+ */
+function perSampleComposite(report, variantKey, ids) {
+    const scores = [];
+    for (const r of report.results) {
+        if (!ids.has(r.sample_id))
+            continue;
+        const v = r.variants[variantKey];
+        if (v && typeof v.compositeScore === 'number')
+            scores.push(v.compositeScore);
+    }
+    return scores;
+}
+/**
+ * The accept decision for one round. With the significance gate on and enough
+ * decision samples, a candidate is accepted only when it is **significantly** above
+ * the current best's fresh re-eval (`bootstrapDiffCI(...).significant && estimate > 0`)
+ * AND its decision score actually beats the recorded best (`pointCand > pointBest`).
+ * The second clause preserves evolve's monotonic invariant: `bestScore` never
+ * decreases. Without it, an unlucky (noise-low) re-eval of the current best could let
+ * a candidate that is significantly above that re-eval — yet still below the recorded
+ * best — win and overwrite the best downward. Off, or under-powered, the gate degrades
+ * to the legacy point-estimate comparison alone. Pure (modulo the seeded bootstrap) so
+ * the core behavior is unit-testable.
+ */
+export function decideAccept(bestScores, candScores, pointBest, pointCand, opts) {
+    const powered = bestScores.length >= MIN_GATE_SAMPLES && candScores.length >= MIN_GATE_SAMPLES;
+    if (opts.significanceGate && powered) {
+        const diff = bootstrapDiffCI(bestScores, candScores, opts.alpha, DEFAULT_BOOTSTRAP_SAMPLES, opts.seed);
+        return {
+            accepted: diff.significant && diff.estimate > 0 && pointCand > pointBest,
+            diffCI: { low: diff.low, high: diff.high, estimate: diff.estimate, significant: diff.significant },
+            underpowered: false,
+        };
+    }
+    return { accepted: pointCand > pointBest, underpowered: opts.significanceGate && !powered };
+}
+/**
+ * A view of `report` whose results are restricted to `sampleIds`. Used to keep the
+ * holdout split out of the sample-fixer: under an active holdout, only training-split
+ * samples may enter the --auto-fix-samples prompt or be rewritten — otherwise the
+ * skill's samples get tuned to the very samples that decide acceptance, reintroducing
+ * the leak holdout exists to prevent.
+ */
+export function restrictReportToSamples(report, sampleIds) {
+    return { ...report, results: report.results.filter((r) => sampleIds.has(r.sample_id)) };
+}
 export function allNonTripwireAssertionsPass(report, variantKey) {
     for (const entry of report.results) {
         const variant = entry.variants[variantKey];
@@ -279,7 +411,36 @@ async function autoFixSamplesAfterSkillRound(opts) {
     }
     return { fixedCount: result.fixedCount, costUSD: result.costUSD };
 }
-export function buildImprovementPrompt(skillContent, score, weakSamples) {
+/** Number of skill lines below which the edit budget never trips — so a tiny skill
+ *  isn't frozen by a percentage threshold that a few lines already blow past. */
+const EDIT_BUDGET_FLOOR_LINES = 10;
+/**
+ * How a candidate differs from the current best, by trimmed non-empty line sets.
+ * `ratio` drives the edit budget; `summary` feeds the rejected-edit memory so the
+ * improver doesn't re-propose changes that already failed. Order-insensitive and
+ * O(n) over small skill files.
+ */
+export function computeEditDelta(before, after, maxSummaryLines = 12) {
+    const beforeArr = before.split('\n').map((l) => l.trim()).filter(Boolean);
+    const afterArr = after.split('\n').map((l) => l.trim()).filter(Boolean);
+    const beforeSet = new Set(beforeArr);
+    const afterSet = new Set(afterArr);
+    const added = [...afterSet].filter((l) => !beforeSet.has(l));
+    const removed = [...beforeSet].filter((l) => !afterSet.has(l));
+    const changedLines = added.length + removed.length;
+    const ratio = changedLines / Math.max(beforeArr.length, 1);
+    const parts = [];
+    for (const l of added.slice(0, maxSummaryLines))
+        parts.push(`+ ${l}`);
+    if (added.length > maxSummaryLines)
+        parts.push(`+ …(其余 +${added.length - maxSummaryLines} 行)`);
+    for (const l of removed.slice(0, maxSummaryLines))
+        parts.push(`- ${l}`);
+    if (removed.length > maxSummaryLines)
+        parts.push(`- …(其余 -${removed.length - maxSummaryLines} 行)`);
+    return { ratio, changedLines, summary: parts.join('\n') || '(无文本差异)' };
+}
+export function buildImprovementPrompt(skillContent, score, weakSamples, rejectedEdits) {
     const weakDetails = weakSamples.map((s) => {
         const parts = [`### ${s.sample_id}（${s.compositeScore}/5.0）`];
         if (s.llmReason)
@@ -308,13 +469,16 @@ export function buildImprovementPrompt(skillContent, score, weakSamples) {
         }
         return parts.join('\n');
     }).join('\n\n');
+    const rejectedSection = rejectedEdits && rejectedEdits.length > 0
+        ? `\n\n## 已试过且未带来显著提升的改法（不要重复）\n\n${rejectedEdits.join('\n\n')}`
+        : '';
     return `## 当前 Skill（平均分: ${score.toFixed(2)}/5.0）
 
 ${skillContent}
 
 ## 低分用例分析
 
-${weakDetails || '（无低分用例）'}`;
+${weakDetails || '（无低分用例）'}${rejectedSection}`;
 }
 function buildImprovementSuffix(mode, candidatePath) {
     if (mode === 'agent') {
@@ -377,7 +541,7 @@ export function mergeEvolveReports(roundReports, skillName, totalCostUSD, sample
     }
     const runId = `evolve-${skillName}-${generateRunId([skillName]).split('-').slice(-2).join('-')}`;
     const report = {
-        kind: 'evaluation',
+        reportKind: 'evaluation',
         id: runId,
         meta: {
             ...firstReport.meta,
@@ -401,7 +565,7 @@ export function mergeEvolveReports(roundReports, skillName, totalCostUSD, sample
     report.analysis = analyzeResults(report, { samples });
     return report;
 }
-export async function evolveSkill({ skillPath, samplesPath, rounds = 5, target = null, stopOnAssertionsPass = false, autoFixSamples = false, sampleFixMaxAttempts = 2, reuseLatestEval = false, model = DEFAULT_MODEL, judgeModels, improveModel = DEFAULT_MODEL, improveMode = 'agent', executorName = 'claude', concurrency = 1, timeoutMs, skipConnectivity = false, effort, noDiagnostic, skipDoctor, onProgress = null, onRoundProgress = null, }) {
+export async function evolveSkill({ skillPath, samplesPath, rounds = 5, target = null, stopOnAssertionsPass = false, autoFixSamples = false, sampleFixMaxAttempts = 2, reuseLatestEval = false, model = DEFAULT_MODEL, judgeModels, improveModel = DEFAULT_MODEL, improveMode = 'agent', executorName = 'claude', concurrency = 1, timeoutMs, skipConnectivity = false, effort, noDiagnostic, skipDoctor, holdoutRatio = 0, significanceGate = true, significanceAlpha = DEFAULT_BOOTSTRAP_ALPHA, testRatio = 0, editBudget = 0.2, rejectMemory = true, onProgress = null, onRoundProgress = null, }) {
     if (judgeModels && judgeModels.length > 1) {
         throw new Error('evolveSkill does not support multi-judge ensemble (received '
             + `${judgeModels.length} judges). Pass a single-judge array, e.g. `
@@ -421,6 +585,72 @@ export async function evolveSkill({ skillPath, samplesPath, rounds = 5, target =
     if (!existsSync(absSamplesPath))
         throw new Error(`samples file not found: ${absSamplesPath}`);
     mkdirSync(evolveDir, { recursive: true });
+    // Split (opt-in). Computed once over the canonical sample order so it's stable
+    // across rounds. `val` drives the accept decision; weak-sample extraction and the
+    // sample-fixer only ever see `train`; `test` is locked away — never seen during the
+    // loop — and read once at the end for an unbiased generalization score. With no
+    // holdout the decision runs on the full set (legacy). A too-small 3-way split
+    // degrades to 2-way, then to full-set.
+    const allSampleIds = loadSamples(absSamplesPath).samples.map((s) => s.sample_id);
+    const threeWay = (testRatio > 0 && holdoutRatio > 0) ? splitTrainValTest(allSampleIds, holdoutRatio, testRatio) : null;
+    const twoWay = (!threeWay && holdoutRatio > 0) ? splitHoldout(allSampleIds, holdoutRatio) : null;
+    const split = threeWay
+        ? { trainIds: threeWay.trainIds, valIds: threeWay.valIds, testIds: threeWay.testIds }
+        : twoWay
+            ? { trainIds: twoWay.trainIds, valIds: twoWay.holdoutIds, testIds: null }
+            : null;
+    const holdoutInfo = holdoutRatio > 0
+        ? {
+            ratio: holdoutRatio,
+            trainCount: split?.trainIds.size ?? allSampleIds.length,
+            holdoutCount: split?.valIds.size ?? 0,
+            ...(split ? {} : { disabled: true }),
+        }
+        : undefined;
+    // test 被请求(配了 --holdout-ratio)但 3-way 太小回退 → 标 disabled，别让用户
+    // 以为拿到了 locked-test 泛化分。
+    const testRequested = testRatio > 0 && holdoutRatio > 0;
+    const testInfo = threeWay
+        ? { ratio: testRatio, count: threeWay.testIds.size }
+        : testRequested ? { ratio: testRatio, count: 0, disabled: true } : undefined;
+    // Deterministic seed so the gate's CIs are reproducible across reruns (and
+    // assertable in tests). Derived from skill identity + sample count, parsed to a uint32.
+    const gateSeed = parseInt(hashString(`${skillName}:${allSampleIds.length}`).slice(0, 8), 16) >>> 0;
+    let gateUnderpowered = false;
+    // Accept-decision score for a report's variant: val composite when a split is
+    // active, otherwise the full-set composite (legacy behavior).
+    const decisionScore = (report, key) => split
+        ? subsetCompositeScore(report, key, split.valIds)
+        : (report.summary[key]?.avgCompositeScore ?? 0);
+    const trainScoreOf = (report, key) => split ? subsetCompositeScore(report, key, split.trainIds) : undefined;
+    // Per-round trajectory tail: train / holdout breakdown, only when split active.
+    const splitScores = (report, key, decision) => split ? { trainScore: trainScoreOf(report, key), holdoutScore: decision } : {};
+    // Unbiased generalization: the best skill's composite on the locked test set,
+    // read once at the very end. Present only under a valid 3-way split; the test
+    // set never influenced selection or weak-sample extraction.
+    const buildGeneralization = () => {
+        if (!testInfo)
+            return {};
+        if (!threeWay || !split?.testIds)
+            return { test: testInfo }; // requested but degraded → disabled, no score
+        const best = roundReports.find((r) => r.round === bestRound)?.report;
+        if (!best)
+            return { test: testInfo };
+        const key = Object.keys(best.summary)[0];
+        return { test: testInfo, generalizationScore: Number(subsetCompositeScore(best, key, split.testIds).toFixed(4)) };
+    };
+    const gateInfo = () => ({ enabled: significanceGate, alpha: significanceAlpha, ...(gateUnderpowered ? { underpowered: true } : {}) });
+    // Rejected-edit memory (most recent K). Fed back into the next round's improvement
+    // prompt so the improver doesn't re-propose changes that already failed to help.
+    const rejectedEdits = [];
+    const REJECT_MEMORY_K = 3;
+    const rememberRejected = (round, summary, reason) => {
+        if (!rejectMemory)
+            return;
+        rejectedEdits.push(`【第 ${round} 轮被拒（${reason}）】\n${summary}`);
+        if (rejectedEdits.length > REJECT_MEMORY_K)
+            rejectedEdits.shift();
+    };
     // Save original as r0
     let currentBest = readFileSync(absSkillPath, 'utf-8').trim();
     const r0Path = join(evolveDir, `${skillName}.r0.md`);
@@ -461,13 +691,13 @@ export async function evolveSkill({ skillPath, samplesPath, rounds = 5, target =
         });
     }
     const baselineVariantKey = Object.keys(baselineReport.summary)[0];
-    bestScore = baselineReport.summary[baselineVariantKey]?.avgCompositeScore ?? 0;
+    bestScore = decisionScore(baselineReport, baselineVariantKey);
     const baselineCost = baselineReused ? 0 : baselineReport.meta.totalCostUSD;
     totalCostUSD += baselineCost;
     const baselineCostReported = baselineReused || !reportHasUnreportedCost(baselineReport);
     if (!baselineCostReported)
         totalCostReported = false;
-    trajectory.push({ round: 0, score: bestScore, delta: 0, accepted: true, costUSD: baselineCost });
+    trajectory.push({ round: 0, score: bestScore, delta: 0, accepted: true, costUSD: baselineCost, ...splitScores(baselineReport, baselineVariantKey, bestScore) });
     roundReports.push({ round: 0, accepted: true, report: baselineReport });
     if (onRoundProgress)
         onRoundProgress({ round: 0, totalRounds: rounds, phase: 'baseline', score: bestScore, costUSD: baselineCost, costReported: baselineCostReported, reused: baselineReused });
@@ -487,6 +717,9 @@ export async function evolveSkill({ skillPath, samplesPath, rounds = 5, target =
             ...(sampleFixes.length > 0 && { sampleFixes }),
             ...(reusedBaselineReportId && { reusedBaselineReportId }),
             ...(totalCostReported ? {} : { costReported: false }),
+            ...(holdoutInfo ? { holdout: holdoutInfo } : {}),
+            ...buildGeneralization(),
+            gate: gateInfo(),
             trajectory,
             bestSkillPath: allVersions[bestRound],
             allVersions,
@@ -513,10 +746,10 @@ export async function evolveSkill({ skillPath, samplesPath, rounds = 5, target =
                 totalCostReported = false;
         }
         const lastVariantKey = Object.keys(lastReport.summary)[0];
-        const weakSamples = extractWeakSamples(lastReport, lastVariantKey);
+        const weakSamples = extractWeakSamples(lastReport, lastVariantKey, 5, split?.trainIds);
         // Generate improvement
         const candidatePath = join(evolveDir, `${skillName}.r${round}.md`);
-        const basePrompt = buildImprovementPrompt(currentBest, bestScore, weakSamples);
+        const basePrompt = buildImprovementPrompt(currentBest, bestScore, weakSamples, rejectMemory ? rejectedEdits : undefined);
         const executor = createExecutor(executorName);
         let candidateContent;
         let improveCostUSD;
@@ -570,12 +803,32 @@ export async function evolveSkill({ skillPath, samplesPath, rounds = 5, target =
         if (!improveCostReported)
             totalCostReported = false;
         allVersions.push(candidatePath);
+        // Edit budget: reject oversized rewrites BEFORE paying for evaluation. The
+        // improvement prompt already asks for ≤ editBudget of lines changed — this
+        // enforces it. A small floor still lets tiny skills change a handful of lines.
+        const editDelta = computeEditDelta(currentBest, candidateContent);
+        if (editBudget > 0 && editDelta.ratio > editBudget && editDelta.changedLines > EDIT_BUDGET_FLOOR_LINES) {
+            totalCostUSD += improveCostUSD;
+            consecutiveRejects++;
+            const reason = `改动过大 ${(editDelta.ratio * 100).toFixed(0)}%（预算 ${(editBudget * 100).toFixed(0)}%），评测前判拒`;
+            rememberRejected(round, editDelta.summary, reason);
+            trajectory.push({ round, score: bestScore, delta: 0, accepted: false, costUSD: improveCostUSD, editRatio: Number(editDelta.ratio.toFixed(4)), rejectedPreEval: true });
+            if (onRoundProgress)
+                onRoundProgress({ round, totalRounds: rounds, phase: 'done', score: bestScore, delta: 0, accepted: false, costUSD: improveCostUSD, costReported: improveCostReported });
+            if (consecutiveRejects >= 2) {
+                stopReason = 'consecutive-rejects';
+                break;
+            }
+            continue;
+        }
         let preEvalSampleFixCost = 0;
         if (autoFixSamples) {
             const sampleFix = await autoFixSamplesAfterSkillRound({
                 samplesPath: absSamplesPath,
                 skillContent: candidateContent,
-                report: lastReport,
+                // Under an active holdout, the sample-fixer may only see training-split samples —
+                // never the holdout samples that drive the accept decision (leak guard).
+                report: split ? restrictReportToSamples(lastReport, split.trainIds) : lastReport,
                 treatmentKey: lastVariantKey,
                 executorName,
                 model: improveModel,
@@ -593,13 +846,28 @@ export async function evolveSkill({ skillPath, samplesPath, rounds = 5, target =
             samplesPath: absSamplesPath, skillDir, model, judgeModels: effectiveJudgeModels, executorName, concurrency, timeoutMs, skipConnectivity, effort, noDiagnostic, skipDoctor, onProgress,
         });
         const candidateVariantKey = Object.keys(candidateReport.summary)[0];
-        const candidateScore = candidateReport.summary[candidateVariantKey]?.avgCompositeScore ?? 0;
+        const candidateScore = decisionScore(candidateReport, candidateVariantKey);
         const roundCost = improveCostUSD + preEvalSampleFixCost + candidateReport.meta.totalCostUSD;
         const roundCostReported = improveCostReported && !reportHasUnreportedCost(candidateReport);
         if (!roundCostReported)
             totalCostReported = false;
         totalCostUSD += improveCostUSD + candidateReport.meta.totalCostUSD;
-        const accepted = candidateScore > bestScore;
+        // Significance accept gate: accept only when the candidate is *significantly*
+        // above the current best on the decision (val) set, not merely numerically higher
+        // — rejecting gains indistinguishable from judge noise. `lastReport` is the current
+        // best's fresh eval and `candidateReport` the candidate's, over the same samples;
+        // bootstrapDiffCI resamples the two arrays independently (conservative — not a paired
+        // bootstrap). Under-powered decision sets degrade to the legacy point-estimate accept
+        // (note: that path compares the prior-round best scalar, not this fresh re-eval) and
+        // flag `gate.underpowered`.
+        const valIds = split ? split.valIds : new Set(allSampleIds);
+        const bestScores = perSampleComposite(lastReport, lastVariantKey, valIds);
+        const candScores = perSampleComposite(candidateReport, candidateVariantKey, valIds);
+        const decision = decideAccept(bestScores, candScores, bestScore, candidateScore, { significanceGate, alpha: significanceAlpha, seed: gateSeed });
+        const accepted = decision.accepted;
+        const diffCI = decision.diffCI;
+        if (decision.underpowered)
+            gateUnderpowered = true;
         if (accepted) {
             currentBest = candidateContent;
             bestScore = candidateScore;
@@ -608,13 +876,15 @@ export async function evolveSkill({ skillPath, samplesPath, rounds = 5, target =
         }
         else {
             consecutiveRejects++;
+            const reason = diffCI ? (diffCI.estimate > 0 ? '提升不显著' : '方向为负') : '未超过当前最优';
+            rememberRejected(round, editDelta.summary, reason);
         }
         if (accepted)
             roundReports.push({ round, accepted, report: candidateReport });
         const roundDelta = candidateScore - trajectory[trajectory.length - 1].score;
-        trajectory.push({ round, score: candidateScore, delta: roundDelta, accepted, costUSD: roundCost });
+        trajectory.push({ round, score: candidateScore, delta: roundDelta, accepted, costUSD: roundCost, ...splitScores(candidateReport, candidateVariantKey, candidateScore), ...(diffCI ? { diffCI } : {}), editRatio: Number(editDelta.ratio.toFixed(4)) });
         if (onRoundProgress)
-            onRoundProgress({ round, totalRounds: rounds, phase: 'done', score: candidateScore, delta: roundDelta, accepted, costUSD: roundCost, costReported: roundCostReported });
+            onRoundProgress({ round, totalRounds: rounds, phase: 'done', score: candidateScore, delta: roundDelta, accepted, costUSD: roundCost, costReported: roundCostReported, ...(diffCI ? { significant: diffCI.significant } : {}) });
         // Early stop
         if (stopOnAssertionsPass && accepted && allNonTripwireAssertionsPass(candidateReport, candidateVariantKey)) {
             stopReason = 'assertions-pass';
@@ -652,6 +922,9 @@ export async function evolveSkill({ skillPath, samplesPath, rounds = 5, target =
         ...(sampleFixes.length > 0 && { sampleFixes }),
         ...(reusedBaselineReportId && { reusedBaselineReportId }),
         ...(totalCostReported ? {} : { costReported: false }),
+        ...(holdoutInfo ? { holdout: holdoutInfo } : {}),
+        ...buildGeneralization(),
+        gate: gateInfo(),
         trajectory,
         bestSkillPath: allVersions[bestRound],
         allVersions,
@@ -678,4 +951,3 @@ async function evaluate(skillFilePath, { samplesPath, skillDir, model, judgeMode
     });
     return report;
 }
-//# sourceMappingURL=evolver.js.map