oh-my-knowledge 0.33.0 → 0.35.0

package/dist/authoring/evolver.d.ts

@@ -21,6 +21,19 @@ interface HoldoutSplit {
     trainIds: Set<string>;
     holdoutIds: Set<string>;
 }
+/** A train / val / test partition. `val` drives the accept decision; `test` is
+ *  locked — never seen during the loop, read once at the end for an unbiased
+ *  generalization score. */
+interface TrainValTestSplit {
+    trainIds: Set<string>;
+    valIds: Set<string>;
+    testIds: Set<string>;
+}
+/** Below this many decision (val) samples the bootstrap diff CI almost never
+ *  excludes 0 for realistic effect sizes, so the significance gate would reject
+ *  every candidate. Under that floor evolve degrades to the point-estimate accept
+ *  and flags `gate.underpowered`. */
+export declare const MIN_GATE_SAMPLES = 8;
 /**
  * Deterministically split sample ids into train / holdout by `ratio` (fraction
  * held out). Holdout members are picked at an even stride so the partition is
@@ -29,6 +42,44 @@ interface HoldoutSplit {
  * MIN_HOLDOUT_SUBSET — the caller then scores on the full set.
  */
 export declare function splitHoldout(sampleIds: string[], ratio: number): HoldoutSplit | null;
+/**
+ * Deterministically split sample ids into train / val / test. `val` is carved
+ * first at an even stride; `test` is carved at an even stride over what remains,
+ * so the three sets are disjoint and stable across rounds/runs (no RNG). Returns
+ * null when either ratio ≤ 0 or any of the three sides would drop below
+ * MIN_HOLDOUT_SUBSET — the caller then degrades to a 2-way (or full-set) split.
+ */
+export declare function splitTrainValTest(sampleIds: string[], valRatio: number, testRatio: number): TrainValTestSplit | null;
+export interface AcceptDecision {
+    accepted: boolean;
+    /** Diff CI (candidate − best) when the gate ran; absent when it degraded. */
+    diffCI?: {
+        low: number;
+        high: number;
+        estimate: number;
+        significant: boolean;
+    };
+    /** True when the gate was requested but the decision set was below MIN_GATE_SAMPLES,
+     *  so the decision degraded to the point-estimate comparison. */
+    underpowered: boolean;
+}
+/**
+ * The accept decision for one round. With the significance gate on and enough
+ * decision samples, a candidate is accepted only when it is **significantly** above
+ * the current best's fresh re-eval (`bootstrapDiffCI(...).significant && estimate > 0`)
+ * AND its decision score actually beats the recorded best (`pointCand > pointBest`).
+ * The second clause preserves evolve's monotonic invariant: `bestScore` never
+ * decreases. Without it, an unlucky (noise-low) re-eval of the current best could let
+ * a candidate that is significantly above that re-eval — yet still below the recorded
+ * best — win and overwrite the best downward. Off, or under-powered, the gate degrades
+ * to the legacy point-estimate comparison alone. Pure (modulo the seeded bootstrap) so
+ * the core behavior is unit-testable.
+ */
+export declare function decideAccept(bestScores: number[], candScores: number[], pointBest: number, pointCand: number, opts: {
+    significanceGate: boolean;
+    alpha: number;
+    seed: number;
+}): AcceptDecision;
 /**
  * A view of `report` whose results are restricted to `sampleIds`. Used to keep the
  * holdout split out of the sample-fixer: under an active holdout, only training-split
@@ -38,7 +89,22 @@ export declare function splitHoldout(sampleIds: string[], ratio: number): Holdou
  */
 export declare function restrictReportToSamples(report: Report, sampleIds: Set<string>): Report;
 export declare function allNonTripwireAssertionsPass(report: Report, variantKey: string): boolean;
-export declare function buildImprovementPrompt(skillContent: string, score: number, weakSamples: WeakSample[]): string;
+interface EditDelta {
+    /** Symmetric line difference (added + removed unique lines) over original line count. */
+    ratio: number;
+    /** Absolute count of added + removed unique lines. */
+    changedLines: number;
+    /** Compact `+`/`-` summary of the changed lines, truncated. */
+    summary: string;
+}
+/**
+ * How a candidate differs from the current best, by trimmed non-empty line sets.
+ * `ratio` drives the edit budget; `summary` feeds the rejected-edit memory so the
+ * improver doesn't re-propose changes that already failed. Order-insensitive and
+ * O(n) over small skill files.
+ */
+export declare function computeEditDelta(before: string, after: string, maxSummaryLines?: number): EditDelta;
+export declare function buildImprovementPrompt(skillContent: string, score: number, weakSamples: WeakSample[], rejectedEdits?: string[]): string;
 /** @deprecated Use ProgressCallback from evaluation-core.ts */
 export type EvolveProgressInfo = Parameters<ProgressCallback>[0];
 export interface EvolveRoundProgressInfo {
@@ -54,6 +120,10 @@ export interface EvolveRoundProgressInfo {
     costReported?: boolean;
     error?: string;
     reused?: boolean;
+    /** When the significance gate ran: whether the candidate's gain was significant.
+     *  False on a rejected round means "score rose but within noise" — lets the CLI
+     *  explain an otherwise-confusing `(+0.0x) ✗ REJECT`. Undefined = gate didn't run. */
+    significant?: boolean;
 }
 interface EvolveOptions {
     skillPath: string;
@@ -88,20 +158,56 @@ interface EvolveOptions {
      *  so the skill is never tuned to the samples that judge it. Too small a split
      *  (either side < MIN_HOLDOUT_SUBSET) falls back to full-set scoring + a warning. */
     holdoutRatio?: number;
+    /** Statistically gate acceptance: a candidate is accepted only when its
+     *  per-sample composite is **significantly** above the current best on the
+     *  decision (val) set — `bootstrapDiffCI(...).significant && estimate > 0` —
+     *  not merely numerically higher. Default true (rejecting improvements
+     *  indistinguishable from judge noise is the point). Below MIN_GATE_SAMPLES
+     *  decision samples the gate is underpowered and degrades to the point-estimate
+     *  comparison + a warning. Set false to force the legacy point-estimate accept. */
+    significanceGate?: boolean;
+    /** Significance level for the accept gate's diff CI. Default 0.05 (95% CI). */
+    significanceAlpha?: number;
+    /** Fraction of samples locked away as a **test** set (0..1). Default 0 = off.
+     *  Only honored alongside `holdoutRatio` > 0 (test needs a separate val set to
+     *  decide on). The test split is never seen during the loop — not by weak-sample
+     *  extraction, not by the accept gate — and is read exactly once at the end for an
+     *  unbiased `generalizationScore`. Too small a 3-way split degrades to 2-way. */
+    testRatio?: number;
+    /** Max fraction of skill lines a single round may change before the candidate is
+     *  rejected **without paying for evaluation**. Default 0.2 (matches the "≤20%"
+     *  the improvement prompt already asks for — this enforces it). A small floor
+     *  always permits a handful of lines so tiny skills aren't frozen. Set 0 to disable. */
+    editBudget?: number;
+    /** Feed rejected candidate edits back into the next round's improvement prompt
+     *  ("these were tried and did not help — don't repeat them"). Default true. */
+    rejectMemory?: boolean;
     onProgress?: ProgressCallback | null;
     onRoundProgress?: ((progress: EvolveRoundProgressInfo) => void) | null;
 }
 interface TrajectoryEntry {
     round: number;
-    /** Accept-decision score: holdout composite when holdout is active, else full-set. */
+    /** Accept-decision score: val composite when a holdout split is active, else full-set. */
     score: number;
     delta: number;
     accepted: boolean;
     costUSD: number;
-    /** Present when holdout is active: the training-split composite (improvement signal). */
+    /** Present when a holdout split is active: the training-split composite (improvement signal). */
     trainScore?: number;
-    /** Present when holdout is active: the holdout-split composite (== score). */
+    /** Present when a holdout split is active: the val-split composite (== score). */
     holdoutScore?: number;
+    /** Significance-gate diff CI (candidate − current best) on the decision set, when
+     *  the gate was powered enough to run. `significant` 决定接受。 */
+    diffCI?: {
+        low: number;
+        high: number;
+        estimate: number;
+        significant: boolean;
+    };
+    /** Fraction of skill lines this candidate changed vs the current best. */
+    editRatio?: number;
+    /** True when the candidate was rejected by the edit budget before evaluation. */
+    rejectedPreEval?: boolean;
 }
 export interface EvolveResult {
     startScore: number;
@@ -127,6 +233,25 @@ export interface EvolveResult {
         holdoutCount: number;
         disabled?: boolean;
     };
+    /** Locked-test split summary. `disabled` is true when `--test-ratio` was requested
+     *  but the 3-way split was too small, so evolve fell back to a 2-way holdout and
+     *  produced no generalization score. */
+    test?: {
+        ratio: number;
+        count: number;
+        disabled?: boolean;
+    };
+    /** Unbiased composite of the best skill on the locked test set — the headline honest
+     *  number. Present only when a 3-way split was active. The test set never influenced
+     *  selection or weak-sample extraction, so this is an out-of-sample estimate. */
+    generalizationScore?: number;
+    /** Accept-gate summary. `underpowered` = the decision set was below MIN_GATE_SAMPLES
+     *  at least once, so the gate degraded to the point-estimate comparison + warned. */
+    gate?: {
+        enabled: boolean;
+        alpha: number;
+        underpowered?: boolean;
+    };
     trajectory: TrajectoryEntry[];
     bestSkillPath: string;
     allVersions: string[];
@@ -138,6 +263,5 @@ export interface RoundReport {
     report: Report;
 }
 export declare function mergeEvolveReports(roundReports: RoundReport[], skillName: string, totalCostUSD: number, samples?: Sample[], skillPath?: string): Report;
-export declare function evolveSkill({ skillPath, samplesPath, rounds, target, stopOnAssertionsPass, autoFixSamples, sampleFixMaxAttempts, reuseLatestEval, model, judgeModels, improveModel, improveMode, executorName, concurrency, timeoutMs, skipConnectivity, effort, noDiagnostic, skipDoctor, holdoutRatio, onProgress, onRoundProgress, }: EvolveOptions): Promise<EvolveResult>;
+export declare function evolveSkill({ skillPath, samplesPath, rounds, target, stopOnAssertionsPass, autoFixSamples, sampleFixMaxAttempts, reuseLatestEval, model, judgeModels, improveModel, improveMode, executorName, concurrency, timeoutMs, skipConnectivity, effort, noDiagnostic, skipDoctor, holdoutRatio, significanceGate, significanceAlpha, testRatio, editBudget, rejectMemory, onProgress, onRoundProgress, }: EvolveOptions): Promise<EvolveResult>;
 export {};
-//# sourceMappingURL=evolver.d.ts.map

package/dist/authoring/evolver.js

@@ -6,7 +6,9 @@ import { persistReport, DEFAULT_OUTPUT_DIR, generateRunId, hashString } from '..
 import { createFileStore } from '../server/report-store.js';
 import { analyzeResults } from '../analysis/report-diagnostics.js';
 import { loadSamples } from '../inputs/load-samples.js';
+import { hashArtifactSource } from '../inputs/content-hash.js';
 import { buildVariantSummary } from '../eval-core/schema.js';
+import { bootstrapDiffCI, DEFAULT_BOOTSTRAP_ALPHA, DEFAULT_BOOTSTRAP_SAMPLES } from '../eval-core/bootstrap.js';
 import { fixSamples } from './sample-fixer.js';
 const IMPROVE_SYSTEM_PROMPT = `你是一个 AI 提示词改进专家。你的任务是分析评测结果中的薄弱环节，针对性地改进 skill（系统提示词），使其在评测中获得更高的分数。
 
@@ -87,9 +89,13 @@ function singleVariantReport(report, variantKey) {
 async function findReusableBaselineReport(opts) {
     const store = createFileStore(DEFAULT_OUTPUT_DIR);
     const { samples } = loadSamples(opts.samplesPath);
-    const artifactHash = hashString(opts.skillContent);
+    const artifactHash = opts.artifactHash;
     const reports = await store.findByArtifactHash(artifactHash);
     for (const report of reports) {
+        // schemaVersion < 2 的报告 artifactHashes 是旧文本哈,与当前树哈不同空间:即便值偶合也不该复用
+        // (口径不同会让 lineage 串错身份)。直接跳过,让旧 baseline 重跑出树哈报告。
+        if ((report.meta.schemaVersion ?? 0) < 2)
+            continue;
         if (report.meta.model !== opts.model || report.meta.executor !== opts.executorName)
             continue;
         if ((report.meta.effort ?? undefined) !== (opts.effort ?? undefined))
@@ -156,9 +162,25 @@ export function extractWeakSamples(report, variantKey, count = 5, sampleIdFilter
         .sort((a, b) => a.compositeScore - b.compositeScore)
         .slice(0, count);
 }
-/** Below this many samples on either side, a holdout split is too small to be
- *  meaningful — evolve falls back to full-set scoring and warns. */
+/** Below this many samples on any side, a split is too small to be meaningful —
+ *  evolve falls back to full-set scoring and warns. */
 const MIN_HOLDOUT_SUBSET = 3;
+/** Below this many decision (val) samples the bootstrap diff CI almost never
+ *  excludes 0 for realistic effect sizes, so the significance gate would reject
+ *  every candidate. Under that floor evolve degrades to the point-estimate accept
+ *  and flags `gate.underpowered`. */
+export const MIN_GATE_SAMPLES = 8;
+/** Pick `count` ids at an even stride across `ids` (deterministic, no RNG) so the
+ *  picked subset is representative of the ordering and stable across rounds/runs. */
+function pickByStride(ids, count) {
+    const picked = new Set();
+    if (count <= 0)
+        return picked;
+    const stride = ids.length / count;
+    for (let k = 0; k < count; k++)
+        picked.add(ids[Math.floor(k * stride)]);
+    return picked;
+}
 /**
  * Deterministically split sample ids into train / holdout by `ratio` (fraction
  * held out). Holdout members are picked at an even stride so the partition is
@@ -173,14 +195,31 @@ export function splitHoldout(sampleIds, ratio) {
     const trainCount = sampleIds.length - holdoutCount;
     if (holdoutCount < MIN_HOLDOUT_SUBSET || trainCount < MIN_HOLDOUT_SUBSET)
         return null;
-    const stride = sampleIds.length / holdoutCount;
-    const holdoutIds = new Set();
-    for (let k = 0; k < holdoutCount; k++) {
-        holdoutIds.add(sampleIds[Math.floor(k * stride)]);
-    }
+    const holdoutIds = pickByStride(sampleIds, holdoutCount);
     const trainIds = new Set(sampleIds.filter((id) => !holdoutIds.has(id)));
     return { trainIds, holdoutIds };
 }
+/**
+ * Deterministically split sample ids into train / val / test. `val` is carved
+ * first at an even stride; `test` is carved at an even stride over what remains,
+ * so the three sets are disjoint and stable across rounds/runs (no RNG). Returns
+ * null when either ratio ≤ 0 or any of the three sides would drop below
+ * MIN_HOLDOUT_SUBSET — the caller then degrades to a 2-way (or full-set) split.
+ */
+export function splitTrainValTest(sampleIds, valRatio, testRatio) {
+    if (!(valRatio > 0) || !(testRatio > 0) || sampleIds.length === 0)
+        return null;
+    const valCount = Math.round(sampleIds.length * valRatio);
+    const testCount = Math.round(sampleIds.length * testRatio);
+    const trainCount = sampleIds.length - valCount - testCount;
+    if (valCount < MIN_HOLDOUT_SUBSET || testCount < MIN_HOLDOUT_SUBSET || trainCount < MIN_HOLDOUT_SUBSET)
+        return null;
+    const valIds = pickByStride(sampleIds, valCount);
+    const remaining = sampleIds.filter((id) => !valIds.has(id));
+    const testIds = pickByStride(remaining, testCount);
+    const trainIds = new Set(sampleIds.filter((id) => !valIds.has(id) && !testIds.has(id)));
+    return { trainIds, valIds, testIds };
+}
 /**
  * Mean composite over the subset of a report's results whose sample_id is in
  * `ids`, using the same aggregation as the full-run summary
@@ -200,6 +239,47 @@ function subsetCompositeScore(report, variantKey, ids) {
         return 0;
     return buildVariantSummary(entries).avgCompositeScore ?? 0;
 }
+/**
+ * Per-sample composite scores over the subset of a report's results whose
+ * sample_id is in `ids`, in result order. Feeds `bootstrapDiffCI` for the
+ * significance accept gate — the array (not the mean) is what the bootstrap
+ * resamples. Entries without a numeric compositeScore are skipped.
+ */
+function perSampleComposite(report, variantKey, ids) {
+    const scores = [];
+    for (const r of report.results) {
+        if (!ids.has(r.sample_id))
+            continue;
+        const v = r.variants[variantKey];
+        if (v && typeof v.compositeScore === 'number')
+            scores.push(v.compositeScore);
+    }
+    return scores;
+}
+/**
+ * The accept decision for one round. With the significance gate on and enough
+ * decision samples, a candidate is accepted only when it is **significantly** above
+ * the current best's fresh re-eval (`bootstrapDiffCI(...).significant && estimate > 0`)
+ * AND its decision score actually beats the recorded best (`pointCand > pointBest`).
+ * The second clause preserves evolve's monotonic invariant: `bestScore` never
+ * decreases. Without it, an unlucky (noise-low) re-eval of the current best could let
+ * a candidate that is significantly above that re-eval — yet still below the recorded
+ * best — win and overwrite the best downward. Off, or under-powered, the gate degrades
+ * to the legacy point-estimate comparison alone. Pure (modulo the seeded bootstrap) so
+ * the core behavior is unit-testable.
+ */
+export function decideAccept(bestScores, candScores, pointBest, pointCand, opts) {
+    const powered = bestScores.length >= MIN_GATE_SAMPLES && candScores.length >= MIN_GATE_SAMPLES;
+    if (opts.significanceGate && powered) {
+        const diff = bootstrapDiffCI(bestScores, candScores, opts.alpha, DEFAULT_BOOTSTRAP_SAMPLES, opts.seed);
+        return {
+            accepted: diff.significant && diff.estimate > 0 && pointCand > pointBest,
+            diffCI: { low: diff.low, high: diff.high, estimate: diff.estimate, significant: diff.significant },
+            underpowered: false,
+        };
+    }
+    return { accepted: pointCand > pointBest, underpowered: opts.significanceGate && !powered };
+}
 /**
  * A view of `report` whose results are restricted to `sampleIds`. Used to keep the
  * holdout split out of the sample-fixer: under an active holdout, only training-split
@@ -336,7 +416,36 @@ async function autoFixSamplesAfterSkillRound(opts) {
     }
     return { fixedCount: result.fixedCount, costUSD: result.costUSD };
 }
-export function buildImprovementPrompt(skillContent, score, weakSamples) {
+/** Number of skill lines below which the edit budget never trips — so a tiny skill
+ *  isn't frozen by a percentage threshold that a few lines already blow past. */
+const EDIT_BUDGET_FLOOR_LINES = 10;
+/**
+ * How a candidate differs from the current best, by trimmed non-empty line sets.
+ * `ratio` drives the edit budget; `summary` feeds the rejected-edit memory so the
+ * improver doesn't re-propose changes that already failed. Order-insensitive and
+ * O(n) over small skill files.
+ */
+export function computeEditDelta(before, after, maxSummaryLines = 12) {
+    const beforeArr = before.split('\n').map((l) => l.trim()).filter(Boolean);
+    const afterArr = after.split('\n').map((l) => l.trim()).filter(Boolean);
+    const beforeSet = new Set(beforeArr);
+    const afterSet = new Set(afterArr);
+    const added = [...afterSet].filter((l) => !beforeSet.has(l));
+    const removed = [...beforeSet].filter((l) => !afterSet.has(l));
+    const changedLines = added.length + removed.length;
+    const ratio = changedLines / Math.max(beforeArr.length, 1);
+    const parts = [];
+    for (const l of added.slice(0, maxSummaryLines))
+        parts.push(`+ ${l}`);
+    if (added.length > maxSummaryLines)
+        parts.push(`+ …(其余 +${added.length - maxSummaryLines} 行)`);
+    for (const l of removed.slice(0, maxSummaryLines))
+        parts.push(`- ${l}`);
+    if (removed.length > maxSummaryLines)
+        parts.push(`- …(其余 -${removed.length - maxSummaryLines} 行)`);
+    return { ratio, changedLines, summary: parts.join('\n') || '(无文本差异)' };
+}
+export function buildImprovementPrompt(skillContent, score, weakSamples, rejectedEdits) {
     const weakDetails = weakSamples.map((s) => {
         const parts = [`### ${s.sample_id}（${s.compositeScore}/5.0）`];
         if (s.llmReason)
@@ -365,13 +474,16 @@ export function buildImprovementPrompt(skillContent, score, weakSamples) {
         }
         return parts.join('\n');
     }).join('\n\n');
+    const rejectedSection = rejectedEdits && rejectedEdits.length > 0
+        ? `\n\n## 已试过且未带来显著提升的改法（不要重复）\n\n${rejectedEdits.join('\n\n')}`
+        : '';
     return `## 当前 Skill（平均分: ${score.toFixed(2)}/5.0）
 
 ${skillContent}
 
 ## 低分用例分析
 
-${weakDetails || '（无低分用例）'}`;
+${weakDetails || '（无低分用例）'}${rejectedSection}`;
 }
 function buildImprovementSuffix(mode, candidatePath) {
     if (mode === 'agent') {
@@ -434,7 +546,7 @@ export function mergeEvolveReports(roundReports, skillName, totalCostUSD, sample
     }
     const runId = `evolve-${skillName}-${generateRunId([skillName]).split('-').slice(-2).join('-')}`;
     const report = {
-        kind: 'evaluation',
+        reportKind: 'evaluation',
         id: runId,
         meta: {
             ...firstReport.meta,
@@ -458,7 +570,7 @@ export function mergeEvolveReports(roundReports, skillName, totalCostUSD, sample
     report.analysis = analyzeResults(report, { samples });
     return report;
 }
-export async function evolveSkill({ skillPath, samplesPath, rounds = 5, target = null, stopOnAssertionsPass = false, autoFixSamples = false, sampleFixMaxAttempts = 2, reuseLatestEval = false, model = DEFAULT_MODEL, judgeModels, improveModel = DEFAULT_MODEL, improveMode = 'agent', executorName = 'claude', concurrency = 1, timeoutMs, skipConnectivity = false, effort, noDiagnostic, skipDoctor, holdoutRatio = 0, onProgress = null, onRoundProgress = null, }) {
+export async function evolveSkill({ skillPath, samplesPath, rounds = 5, target = null, stopOnAssertionsPass = false, autoFixSamples = false, sampleFixMaxAttempts = 2, reuseLatestEval = false, model = DEFAULT_MODEL, judgeModels, improveModel = DEFAULT_MODEL, improveMode = 'agent', executorName = 'claude', concurrency = 1, timeoutMs, skipConnectivity = false, effort, noDiagnostic, skipDoctor, holdoutRatio = 0, significanceGate = true, significanceAlpha = DEFAULT_BOOTSTRAP_ALPHA, testRatio = 0, editBudget = 0.2, rejectMemory = true, onProgress = null, onRoundProgress = null, }) {
     if (judgeModels && judgeModels.length > 1) {
         throw new Error('evolveSkill does not support multi-judge ensemble (received '
             + `${judgeModels.length} judges). Pass a single-judge array, e.g. `
@@ -478,28 +590,72 @@ export async function evolveSkill({ skillPath, samplesPath, rounds = 5, target =
     if (!existsSync(absSamplesPath))
         throw new Error(`samples file not found: ${absSamplesPath}`);
     mkdirSync(evolveDir, { recursive: true });
-    // Holdout split (opt-in). Computed once over the canonical sample order so it's
-    // stable across rounds. When active, accept decisions use the holdout composite
-    // and weak-sample extraction only sees the training split — the skill is never
-    // tuned to the samples that decide whether it's accepted.
+    // Split (opt-in). Computed once over the canonical sample order so it's stable
+    // across rounds. `val` drives the accept decision; weak-sample extraction and the
+    // sample-fixer only ever see `train`; `test` is locked away — never seen during the
+    // loop — and read once at the end for an unbiased generalization score. With no
+    // holdout the decision runs on the full set (legacy). A too-small 3-way split
+    // degrades to 2-way, then to full-set.
     const allSampleIds = loadSamples(absSamplesPath).samples.map((s) => s.sample_id);
-    const holdoutSplit = splitHoldout(allSampleIds, holdoutRatio);
+    const threeWay = (testRatio > 0 && holdoutRatio > 0) ? splitTrainValTest(allSampleIds, holdoutRatio, testRatio) : null;
+    const twoWay = (!threeWay && holdoutRatio > 0) ? splitHoldout(allSampleIds, holdoutRatio) : null;
+    const split = threeWay
+        ? { trainIds: threeWay.trainIds, valIds: threeWay.valIds, testIds: threeWay.testIds }
+        : twoWay
+            ? { trainIds: twoWay.trainIds, valIds: twoWay.holdoutIds, testIds: null }
+            : null;
     const holdoutInfo = holdoutRatio > 0
         ? {
             ratio: holdoutRatio,
-            trainCount: holdoutSplit?.trainIds.size ?? allSampleIds.length,
-            holdoutCount: holdoutSplit?.holdoutIds.size ?? 0,
-            ...(holdoutSplit ? {} : { disabled: true }),
+            trainCount: split?.trainIds.size ?? allSampleIds.length,
+            holdoutCount: split?.valIds.size ?? 0,
+            ...(split ? {} : { disabled: true }),
         }
         : undefined;
-    // Accept-decision score for a report's variant: holdout composite when the split
-    // is active, otherwise the full-set composite (legacy behavior).
-    const decisionScore = (report, key) => holdoutSplit
-        ? subsetCompositeScore(report, key, holdoutSplit.holdoutIds)
+    // test 被请求(配了 --holdout-ratio)但 3-way 太小回退 → 标 disabled，别让用户
+    // 以为拿到了 locked-test 泛化分。
+    const testRequested = testRatio > 0 && holdoutRatio > 0;
+    const testInfo = threeWay
+        ? { ratio: testRatio, count: threeWay.testIds.size }
+        : testRequested ? { ratio: testRatio, count: 0, disabled: true } : undefined;
+    // Deterministic seed so the gate's CIs are reproducible across reruns (and
+    // assertable in tests). Derived from skill identity + sample count, parsed to a uint32.
+    const gateSeed = parseInt(hashString(`${skillName}:${allSampleIds.length}`).slice(0, 8), 16) >>> 0;
+    let gateUnderpowered = false;
+    // Accept-decision score for a report's variant: val composite when a split is
+    // active, otherwise the full-set composite (legacy behavior).
+    const decisionScore = (report, key) => split
+        ? subsetCompositeScore(report, key, split.valIds)
         : (report.summary[key]?.avgCompositeScore ?? 0);
-    const trainScoreOf = (report, key) => holdoutSplit ? subsetCompositeScore(report, key, holdoutSplit.trainIds) : undefined;
+    const trainScoreOf = (report, key) => split ? subsetCompositeScore(report, key, split.trainIds) : undefined;
     // Per-round trajectory tail: train / holdout breakdown, only when split active.
-    const splitScores = (report, key, decision) => holdoutSplit ? { trainScore: trainScoreOf(report, key), holdoutScore: decision } : {};
+    const splitScores = (report, key, decision) => split ? { trainScore: trainScoreOf(report, key), holdoutScore: decision } : {};
+    // Unbiased generalization: the best skill's composite on the locked test set,
+    // read once at the very end. Present only under a valid 3-way split; the test
+    // set never influenced selection or weak-sample extraction.
+    const buildGeneralization = () => {
+        if (!testInfo)
+            return {};
+        if (!threeWay || !split?.testIds)
+            return { test: testInfo }; // requested but degraded → disabled, no score
+        const best = roundReports.find((r) => r.round === bestRound)?.report;
+        if (!best)
+            return { test: testInfo };
+        const key = Object.keys(best.summary)[0];
+        return { test: testInfo, generalizationScore: Number(subsetCompositeScore(best, key, split.testIds).toFixed(4)) };
+    };
+    const gateInfo = () => ({ enabled: significanceGate, alpha: significanceAlpha, ...(gateUnderpowered ? { underpowered: true } : {}) });
+    // Rejected-edit memory (most recent K). Fed back into the next round's improvement
+    // prompt so the improver doesn't re-propose changes that already failed to help.
+    const rejectedEdits = [];
+    const REJECT_MEMORY_K = 3;
+    const rememberRejected = (round, summary, reason) => {
+        if (!rejectMemory)
+            return;
+        rejectedEdits.push(`【第 ${round} 轮被拒（${reason}）】\n${summary}`);
+        if (rejectedEdits.length > REJECT_MEMORY_K)
+            rejectedEdits.shift();
+    };
     // Save original as r0
     let currentBest = readFileSync(absSkillPath, 'utf-8').trim();
     const r0Path = join(evolveDir, `${skillName}.r0.md`);
@@ -518,10 +674,14 @@ export async function evolveSkill({ skillPath, samplesPath, rounds = 5, target =
     let stopReason = 'rounds';
     // 给定一个 report 看任一 variant 的 exec/judge cost 是否未报告
     const reportHasUnreportedCost = (rep) => Object.values(rep.summary).some((v) => v.execCostReported === false || v.judgeCostReported === false);
-    // Round 0: baseline evaluation
+    // Round 0: baseline evaluation。复用查询键走整树哈,与 eval 报告口径一致:round 0 的 currentBest 即
+    // 磁盘上原始 skill,树哈取自磁盘(dir-skill 哈整目录、单文件 .md 哈单文件)。evolve 只改 SKILL.md 正文、
+    // 不动 references/ 资产,故磁盘树哈即该 baseline 的权威指纹。
+    const baselineIsDirSkill = basename(absSkillPath) === 'SKILL.md';
+    const baselineArtifactHash = hashArtifactSource(baselineIsDirSkill ? skillDir : absSkillPath, baselineIsDirSkill);
     let baselineReport = reuseLatestEval
         ? await findReusableBaselineReport({
-            skillContent: currentBest,
+            artifactHash: baselineArtifactHash,
             samplesPath: absSamplesPath,
             model,
             executorName,
@@ -567,6 +727,8 @@ export async function evolveSkill({ skillPath, samplesPath, rounds = 5, target =
             ...(reusedBaselineReportId && { reusedBaselineReportId }),
             ...(totalCostReported ? {} : { costReported: false }),
             ...(holdoutInfo ? { holdout: holdoutInfo } : {}),
+            ...buildGeneralization(),
+            gate: gateInfo(),
             trajectory,
             bestSkillPath: allVersions[bestRound],
             allVersions,
@@ -593,10 +755,10 @@ export async function evolveSkill({ skillPath, samplesPath, rounds = 5, target =
                 totalCostReported = false;
         }
         const lastVariantKey = Object.keys(lastReport.summary)[0];
-        const weakSamples = extractWeakSamples(lastReport, lastVariantKey, 5, holdoutSplit?.trainIds);
+        const weakSamples = extractWeakSamples(lastReport, lastVariantKey, 5, split?.trainIds);
         // Generate improvement
         const candidatePath = join(evolveDir, `${skillName}.r${round}.md`);
-        const basePrompt = buildImprovementPrompt(currentBest, bestScore, weakSamples);
+        const basePrompt = buildImprovementPrompt(currentBest, bestScore, weakSamples, rejectMemory ? rejectedEdits : undefined);
         const executor = createExecutor(executorName);
         let candidateContent;
         let improveCostUSD;
@@ -650,6 +812,24 @@ export async function evolveSkill({ skillPath, samplesPath, rounds = 5, target =
         if (!improveCostReported)
             totalCostReported = false;
         allVersions.push(candidatePath);
+        // Edit budget: reject oversized rewrites BEFORE paying for evaluation. The
+        // improvement prompt already asks for ≤ editBudget of lines changed — this
+        // enforces it. A small floor still lets tiny skills change a handful of lines.
+        const editDelta = computeEditDelta(currentBest, candidateContent);
+        if (editBudget > 0 && editDelta.ratio > editBudget && editDelta.changedLines > EDIT_BUDGET_FLOOR_LINES) {
+            totalCostUSD += improveCostUSD;
+            consecutiveRejects++;
+            const reason = `改动过大 ${(editDelta.ratio * 100).toFixed(0)}%（预算 ${(editBudget * 100).toFixed(0)}%），评测前判拒`;
+            rememberRejected(round, editDelta.summary, reason);
+            trajectory.push({ round, score: bestScore, delta: 0, accepted: false, costUSD: improveCostUSD, editRatio: Number(editDelta.ratio.toFixed(4)), rejectedPreEval: true });
+            if (onRoundProgress)
+                onRoundProgress({ round, totalRounds: rounds, phase: 'done', score: bestScore, delta: 0, accepted: false, costUSD: improveCostUSD, costReported: improveCostReported });
+            if (consecutiveRejects >= 2) {
+                stopReason = 'consecutive-rejects';
+                break;
+            }
+            continue;
+        }
         let preEvalSampleFixCost = 0;
         if (autoFixSamples) {
             const sampleFix = await autoFixSamplesAfterSkillRound({
@@ -657,7 +837,7 @@ export async function evolveSkill({ skillPath, samplesPath, rounds = 5, target =
                 skillContent: candidateContent,
                 // Under an active holdout, the sample-fixer may only see training-split samples —
                 // never the holdout samples that drive the accept decision (leak guard).
-                report: holdoutSplit ? restrictReportToSamples(lastReport, holdoutSplit.trainIds) : lastReport,
+                report: split ? restrictReportToSamples(lastReport, split.trainIds) : lastReport,
                 treatmentKey: lastVariantKey,
                 executorName,
                 model: improveModel,
@@ -681,7 +861,22 @@ export async function evolveSkill({ skillPath, samplesPath, rounds = 5, target =
         if (!roundCostReported)
             totalCostReported = false;
         totalCostUSD += improveCostUSD + candidateReport.meta.totalCostUSD;
-        const accepted = candidateScore > bestScore;
+        // Significance accept gate: accept only when the candidate is *significantly*
+        // above the current best on the decision (val) set, not merely numerically higher
+        // — rejecting gains indistinguishable from judge noise. `lastReport` is the current
+        // best's fresh eval and `candidateReport` the candidate's, over the same samples;
+        // bootstrapDiffCI resamples the two arrays independently (conservative — not a paired
+        // bootstrap). Under-powered decision sets degrade to the legacy point-estimate accept
+        // (note: that path compares the prior-round best scalar, not this fresh re-eval) and
+        // flag `gate.underpowered`.
+        const valIds = split ? split.valIds : new Set(allSampleIds);
+        const bestScores = perSampleComposite(lastReport, lastVariantKey, valIds);
+        const candScores = perSampleComposite(candidateReport, candidateVariantKey, valIds);
+        const decision = decideAccept(bestScores, candScores, bestScore, candidateScore, { significanceGate, alpha: significanceAlpha, seed: gateSeed });
+        const accepted = decision.accepted;
+        const diffCI = decision.diffCI;
+        if (decision.underpowered)
+            gateUnderpowered = true;
         if (accepted) {
             currentBest = candidateContent;
             bestScore = candidateScore;
@@ -690,13 +885,15 @@ export async function evolveSkill({ skillPath, samplesPath, rounds = 5, target =
         }
         else {
             consecutiveRejects++;
+            const reason = diffCI ? (diffCI.estimate > 0 ? '提升不显著' : '方向为负') : '未超过当前最优';
+            rememberRejected(round, editDelta.summary, reason);
         }
         if (accepted)
             roundReports.push({ round, accepted, report: candidateReport });
         const roundDelta = candidateScore - trajectory[trajectory.length - 1].score;
-        trajectory.push({ round, score: candidateScore, delta: roundDelta, accepted, costUSD: roundCost, ...splitScores(candidateReport, candidateVariantKey, candidateScore) });
+        trajectory.push({ round, score: candidateScore, delta: roundDelta, accepted, costUSD: roundCost, ...splitScores(candidateReport, candidateVariantKey, candidateScore), ...(diffCI ? { diffCI } : {}), editRatio: Number(editDelta.ratio.toFixed(4)) });
         if (onRoundProgress)
-            onRoundProgress({ round, totalRounds: rounds, phase: 'done', score: candidateScore, delta: roundDelta, accepted, costUSD: roundCost, costReported: roundCostReported });
+            onRoundProgress({ round, totalRounds: rounds, phase: 'done', score: candidateScore, delta: roundDelta, accepted, costUSD: roundCost, costReported: roundCostReported, ...(diffCI ? { significant: diffCI.significant } : {}) });
         // Early stop
         if (stopOnAssertionsPass && accepted && allNonTripwireAssertionsPass(candidateReport, candidateVariantKey)) {
             stopReason = 'assertions-pass';
@@ -735,6 +932,8 @@ export async function evolveSkill({ skillPath, samplesPath, rounds = 5, target =
         ...(reusedBaselineReportId && { reusedBaselineReportId }),
         ...(totalCostReported ? {} : { costReported: false }),
         ...(holdoutInfo ? { holdout: holdoutInfo } : {}),
+        ...buildGeneralization(),
+        gate: gateInfo(),
         trajectory,
         bestSkillPath: allVersions[bestRound],
         allVersions,
@@ -761,4 +960,3 @@ async function evaluate(skillFilePath, { samplesPath, skillDir, model, judgeMode
     });
     return report;
 }
-//# sourceMappingURL=evolver.js.map

package/dist/authoring/generator.d.ts

@@ -63,4 +63,3 @@ export declare function sanitizeGeneratedSamples(samples: Sample[], opts?: {
     stripped: string[];
 };
 export {};
-//# sourceMappingURL=generator.d.ts.map

package/dist/authoring/generator.js

@@ -830,4 +830,3 @@ export function sanitizeGeneratedSamples(samples, opts = {}) {
     }
     return { stripped };
 }
-//# sourceMappingURL=generator.js.map

package/dist/authoring/sample-fixer.d.ts

@@ -48,4 +48,3 @@ export interface FixSamplesResult {
     }>;
 }
 export declare function fixSamples(options: FixSamplesOptions): Promise<FixSamplesResult>;
-//# sourceMappingURL=sample-fixer.d.ts.map

package/dist/authoring/sample-fixer.js

@@ -210,4 +210,3 @@ ${sampleSections}
         };
     }
 }
-//# sourceMappingURL=sample-fixer.js.map

package/dist/cli/commands/doctor.d.ts

@@ -25,4 +25,3 @@ export default class Doctor extends BaseCommand {
     run(): Promise<void>;
 }
 export declare function pruneDoctorHistory(dir: string, skillName: string, maxKeep: number): void;
-//# sourceMappingURL=doctor.d.ts.map