oh-my-knowledge 0.31.0 → 0.33.0

package/dist/authoring/evolver.js

@@ -0,0 +1,764 @@
+import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs';
+import { resolve, join, dirname, basename } from 'node:path';
+import { runEvaluation } from '../eval-workflows/run-evaluation.js';
+import { createExecutor, DEFAULT_MODEL, JUDGE_MODEL } from '../executors/index.js';
+import { persistReport, DEFAULT_OUTPUT_DIR, generateRunId, hashString } from '../eval-core/evaluation-reporting.js';
+import { createFileStore } from '../server/report-store.js';
+import { analyzeResults } from '../analysis/report-diagnostics.js';
+import { loadSamples } from '../inputs/load-samples.js';
+import { buildVariantSummary } from '../eval-core/schema.js';
+import { fixSamples } from './sample-fixer.js';
+const IMPROVE_SYSTEM_PROMPT = `你是一个 AI 提示词改进专家。你的任务是分析评测结果中的薄弱环节，针对性地改进 skill（系统提示词），使其在评测中获得更高的分数。
+
+改进原则：
+1. 针对低分用例暴露的具体问题做改进，不要泛泛修改
+2. 保留当前版本中已经表现良好的部分
+3. 保持 skill 的整体结构和格式
+4. 改进应该具体、可执行，不要空泛的描述
+5. 改动必须最小化：只修改或新增与低分用例直接相关的段落，不要重写未出问题的部分
+6. 禁止重新组织整篇结构、格式或措辞；保留原文中未出问题的段落原封不动
+7. 每轮改动行数不超过原文的 20%
+
+直接输出改进后的 skill 内容，不要包含 markdown 代码块标记或任何解释说明。`;
+const IMPROVE_AGENT_SYSTEM_PROMPT = `你是一个 AI 提示词改进专家。你的任务是分析评测结果中的薄弱环节，使用 Edit 工具针对性地修改 skill 文件。
+
+改进原则：
+1. 只修改与低分用例直接相关的段落，不要动已经表现良好的部分
+2. 使用 Edit 工具做最小化修改，不要重写整篇文件
+3. 每次 Edit 只改一处具体问题，可以多次 Edit
+4. 不要重新组织文档结构、不要调整格式、不要修改措辞风格
+5. 新增内容应紧跟在相关现有段落之后，不要大段插入
+6. 改进应该具体、可执行，不要空泛的描述
+7. 修改完成后不要输出文件全文，只说明改了什么`;
+/**
+ * Read the `name` field from a SKILL.md frontmatter.
+ * Returns null if the file has no frontmatter or no name field.
+ * Used to give evolve reports semantic filenames instead of "evolve-SKILL-xxx".
+ */
+function canonicalStringify(value) {
+    if (value === null || typeof value !== 'object')
+        return JSON.stringify(value);
+    if (Array.isArray(value))
+        return '[' + value.map(canonicalStringify).join(',') + ']';
+    const entries = Object.keys(value).sort();
+    return '{' + entries.map((k) => JSON.stringify(k) + ':' + canonicalStringify(value[k])).join(',') + '}';
+}
+function hashSampleForReuse(sample) {
+    return hashString(canonicalStringify({
+        prompt: sample.prompt,
+        rubric: sample.rubric ?? null,
+        dimensions: sample.dimensions ?? null,
+        assertions: sample.assertions ?? null,
+        schema: sample.schema ?? null,
+    }));
+}
+function sameJudgeModels(report, judges) {
+    const metaJudges = report.meta.judgeModels ?? [];
+    if (metaJudges.length !== judges.length)
+        return false;
+    return metaJudges.every((j, i) => j.executor === judges[i].executor && j.model === judges[i].model);
+}
+function sampleHashesMatch(report, samples) {
+    const hashes = report.meta.sampleHashes;
+    if (!hashes)
+        return false;
+    return samples.every((s) => hashes[s.sample_id] === hashSampleForReuse(s));
+}
+function singleVariantReport(report, variantKey) {
+    const summary = report.summary[variantKey];
+    return {
+        ...report,
+        meta: {
+            ...report.meta,
+            variants: [variantKey],
+            artifactHashes: report.meta.artifactHashes?.[variantKey]
+                ? { [variantKey]: report.meta.artifactHashes[variantKey] }
+                : {},
+            ...(report.meta.variantConfigs ? { variantConfigs: report.meta.variantConfigs.filter((cfg) => cfg.variant === variantKey) } : {}),
+            ...(report.meta.skillIsolation ? { skillIsolation: { [variantKey]: report.meta.skillIsolation[variantKey] ?? null } } : {}),
+        },
+        summary: { [variantKey]: summary },
+        results: report.results.map((entry) => ({
+            sample_id: entry.sample_id,
+            variants: entry.variants[variantKey] ? { [variantKey]: entry.variants[variantKey] } : {},
+        })),
+    };
+}
+async function findReusableBaselineReport(opts) {
+    const store = createFileStore(DEFAULT_OUTPUT_DIR);
+    const { samples } = loadSamples(opts.samplesPath);
+    const artifactHash = hashString(opts.skillContent);
+    const reports = await store.findByArtifactHash(artifactHash);
+    for (const report of reports) {
+        if (report.meta.model !== opts.model || report.meta.executor !== opts.executorName)
+            continue;
+        if ((report.meta.effort ?? undefined) !== (opts.effort ?? undefined))
+            continue;
+        if (report.meta.noJudge)
+            continue;
+        if (report.meta.budgetExhausted)
+            continue;
+        if (!sameJudgeModels(report, opts.judgeModels))
+            continue;
+        if (!sampleHashesMatch(report, samples))
+            continue;
+        const variantKey = report.meta.variants.find((name) => report.meta.artifactHashes?.[name] === artifactHash);
+        if (!variantKey || !report.summary[variantKey])
+            continue;
+        return singleVariantReport(report, variantKey);
+    }
+    return null;
+}
+function readSkillName(skillPath) {
+    try {
+        const content = readFileSync(skillPath, 'utf-8');
+        const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---/);
+        if (!match)
+            return null;
+        const nameMatch = match[1].match(/^name:\s*["']?([^"'\r\n]+?)["']?\s*$/m);
+        return nameMatch ? nameMatch[1].trim() : null;
+    }
+    catch {
+        return null;
+    }
+}
+export function extractWeakSamples(report, variantKey, count = 5, sampleIdFilter) {
+    const weakSamples = [];
+    for (const r of report.results) {
+        if (sampleIdFilter && !sampleIdFilter.has(r.sample_id))
+            continue;
+        const v = r.variants[variantKey];
+        if (!v || typeof v.compositeScore !== 'number')
+            continue;
+        const suggestion = v.diagnostic?.suggestion;
+        weakSamples.push({
+            sample_id: r.sample_id,
+            compositeScore: v.compositeScore,
+            llmReason: v.llmReason || null,
+            failedAssertions: v.assertions?.details?.filter((a) => !a.passed).map((a) => `${a.type}: ${a.value ?? ''}`) || [],
+            dimensions: v.dimensions
+                ? Object.fromEntries(Object.entries(v.dimensions).map(([k, info]) => [k, typeof info === 'object' ? info.score : info]))
+                : null,
+            ...(v.diagnostic ? {
+                diagnostic: {
+                    summary: v.diagnostic.summary,
+                    expected: v.diagnostic.expected,
+                    actual: v.diagnostic.actual,
+                    rootCause: v.diagnostic.rootCause,
+                    skillSuggestion: suggestion?.skill,
+                    sampleSuggestion: suggestion?.sample,
+                    none: suggestion?.none,
+                },
+            } : {}),
+        });
+    }
+    return weakSamples
+        .sort((a, b) => a.compositeScore - b.compositeScore)
+        .slice(0, count);
+}
+/** Below this many samples on either side, a holdout split is too small to be
+ *  meaningful — evolve falls back to full-set scoring and warns. */
+const MIN_HOLDOUT_SUBSET = 3;
+/**
+ * Deterministically split sample ids into train / holdout by `ratio` (fraction
+ * held out). Holdout members are picked at an even stride so the partition is
+ * representative of the ordering, and the split is stable across rounds and runs
+ * (no RNG). Returns null when ratio ≤ 0 or either side would drop below
+ * MIN_HOLDOUT_SUBSET — the caller then scores on the full set.
+ */
+export function splitHoldout(sampleIds, ratio) {
+    if (!(ratio > 0) || sampleIds.length === 0)
+        return null;
+    const holdoutCount = Math.round(sampleIds.length * ratio);
+    const trainCount = sampleIds.length - holdoutCount;
+    if (holdoutCount < MIN_HOLDOUT_SUBSET || trainCount < MIN_HOLDOUT_SUBSET)
+        return null;
+    const stride = sampleIds.length / holdoutCount;
+    const holdoutIds = new Set();
+    for (let k = 0; k < holdoutCount; k++) {
+        holdoutIds.add(sampleIds[Math.floor(k * stride)]);
+    }
+    const trainIds = new Set(sampleIds.filter((id) => !holdoutIds.has(id)));
+    return { trainIds, holdoutIds };
+}
+/**
+ * Mean composite over the subset of a report's results whose sample_id is in
+ * `ids`, using the same aggregation as the full-run summary
+ * (`buildVariantSummary`) so train / holdout scores stay comparable to the
+ * headline composite. Returns 0 when the subset has no scorable entries.
+ */
+function subsetCompositeScore(report, variantKey, ids) {
+    const entries = [];
+    for (const r of report.results) {
+        if (!ids.has(r.sample_id))
+            continue;
+        const v = r.variants[variantKey];
+        if (v)
+            entries.push(v);
+    }
+    if (entries.length === 0)
+        return 0;
+    return buildVariantSummary(entries).avgCompositeScore ?? 0;
+}
+/**
+ * A view of `report` whose results are restricted to `sampleIds`. Used to keep the
+ * holdout split out of the sample-fixer: under an active holdout, only training-split
+ * samples may enter the --auto-fix-samples prompt or be rewritten — otherwise the
+ * skill's samples get tuned to the very samples that decide acceptance, reintroducing
+ * the leak holdout exists to prevent.
+ */
+export function restrictReportToSamples(report, sampleIds) {
+    return { ...report, results: report.results.filter((r) => sampleIds.has(r.sample_id)) };
+}
+export function allNonTripwireAssertionsPass(report, variantKey) {
+    for (const entry of report.results) {
+        const variant = entry.variants[variantKey];
+        if (!variant || variant.ok === false)
+            return false;
+        const snapshotTripwire = report.sampleSnapshots?.[entry.sample_id]?.tripwire === true;
+        const diagnosticTripwire = (variant.diagnostic?.rootCause ?? []).includes('tripwire_intentional');
+        if (snapshotTripwire || diagnosticTripwire)
+            continue;
+        const details = variant.assertions?.details ?? [];
+        if (!details.every((d) => d.passed))
+            return false;
+    }
+    return true;
+}
+const FIX_AGENT_SYSTEM_PROMPT = `你是一个评测用例修复专家。你的任务是使用 Edit 工具直接修改 samples JSON 文件中有问题的用例。
+
+修复原则：
+1. 先分析失败原因——是 sample 设计问题（mock 缺失 / 断言写错 / 工具名不匹配）还是 LLM 行为问题（LLM 真的做错了,低分合理）
+2. 如果是 LLM 行为问题（低分合理）,不要改 sample
+3. 如果是 sample 设计问题,用 Edit 工具修复对应 sample 的 mocks / assertions / environment
+4. 保持 sample_id 不变,保持测试意图不变
+5. 每次 Edit 只改一个 sample 的一处问题,可以多次 Edit
+6. 不要重写整个 JSON 文件,只改有问题的字段
+7. 断言修复只保留核心行为节点,不要过度绑定实现细节
+8. 不要删除整条用例
+9. 只能修改 sample,绝对不能建议修改 SKILL.md`;
+function createSampleFixExecutor(executorName) {
+    const executor = createExecutor(executorName);
+    return async (opts) => {
+        const result = await executor({
+            model: opts.model,
+            system: opts.system,
+            prompt: opts.prompt,
+            timeoutMs: opts.timeoutMs,
+            lean: opts.lean,
+            ...(opts.cwd && { cwd: opts.cwd }),
+        });
+        return { ok: result.ok, text: result.output ?? '', costUSD: result.costUSD };
+    };
+}
+async function autoFixSamplesAgent(opts) {
+    const { samples } = loadSamples(opts.samplesPath);
+    const sampleMap = new Map(samples.map((s) => [s.sample_id, s]));
+    const fixContexts = [];
+    for (const entry of opts.report.results) {
+        const variant = entry.variants?.[opts.treatmentKey];
+        if (!variant)
+            continue;
+        const variantObj = variant;
+        if (variantObj.ok === false)
+            continue;
+        const assertionDetails = variantObj.assertions?.details ?? [];
+        if (assertionDetails.length === 0 || assertionDetails.every((d) => d.passed))
+            continue;
+        const sid = entry.sample_id;
+        const original = sampleMap.get(sid);
+        if (!original)
+            continue;
+        if (typeof original.omkFix === 'object') {
+            const attempts = original.omkFix?.attempts;
+            if (typeof attempts === 'number' && attempts >= opts.maxAttemptsPerSample)
+                continue;
+        }
+        const diag = variantObj.diagnostic;
+        const toolCalls = (variantObj.toolCalls ?? []);
+        const failedList = assertionDetails.filter((a) => !a.passed).map((a) => `${a.type}: ${a.value}`).join('\n');
+        const toolSummary = toolCalls.length > 0
+            ? toolCalls.map((tc, i) => `[${i}] ${tc.tool} success=${tc.success}`).join('\n')
+            : '(无工具调用)';
+        fixContexts.push({
+            sampleId: sid,
+            diag: `归因: ${(diag?.rootCause ?? []).join(', ') || '(无)'}\n摘要: ${diag?.summary ?? ''}\n期望: ${diag?.expected ?? ''}\n实际: ${diag?.actual ?? ''}\n建议: ${(diag?.suggestion?.sample) ?? '(无)'}`,
+            failedAssertions: `失败断言:\n${failedList}\n\n实际工具调用:\n${toolSummary}`,
+        });
+    }
+    if (fixContexts.length === 0)
+        return { fixedCount: 0, costUSD: 0 };
+    const skillPreview = opts.skillContent.length > 4000
+        ? opts.skillContent.slice(0, 4000) + '\n\n... (truncated)'
+        : opts.skillContent;
+    const sampleSections = fixContexts.map((ctx) => `### ${ctx.sampleId}\n\n${ctx.diag}\n\n${ctx.failedAssertions}`).join('\n\n---\n\n');
+    const prompt = `以下有 ${fixContexts.length} 条失败的评测用例需要分析修复。
+
+## Skill 原文（参考，不可修改）
+
+${skillPreview}
+
+## 待修复的用例
+
+${sampleSections}
+
+请使用 Edit 工具修改文件 ${opts.samplesPath}，只改有问题的 sample 的 assertions / mocks / environment 字段。如果判断是 LLM 行为问题（低分合理），不要改该 sample。`;
+    const executor = createExecutor(opts.executorName);
+    const beforeContent = readFileSync(opts.samplesPath, 'utf-8');
+    const result = await executor({
+        model: opts.model,
+        system: FIX_AGENT_SYSTEM_PROMPT,
+        prompt,
+        cwd: dirname(opts.samplesPath),
+        timeoutMs: 300_000,
+    });
+    const afterContent = readFileSync(opts.samplesPath, 'utf-8');
+    const changed = afterContent !== beforeContent;
+    const fixedCount = changed ? fixContexts.length : 0;
+    return { fixedCount, costUSD: result.costUSD };
+}
+async function autoFixSamplesAfterSkillRound(opts) {
+    const { samples } = loadSamples(opts.samplesPath);
+    if (opts.improveMode === 'agent') {
+        return autoFixSamplesAgent(opts);
+    }
+    const result = await fixSamples({
+        skillContent: opts.skillContent,
+        samples: samples,
+        report: opts.report,
+        treatmentKey: opts.treatmentKey,
+        executor: createSampleFixExecutor(opts.executorName),
+        model: opts.model,
+        maxAttemptsPerSample: opts.maxAttemptsPerSample,
+    });
+    if (result.fixedCount > 0) {
+        writeFileSync(opts.samplesPath, JSON.stringify(result.samples, null, 2));
+    }
+    return { fixedCount: result.fixedCount, costUSD: result.costUSD };
+}
+export function buildImprovementPrompt(skillContent, score, weakSamples) {
+    const weakDetails = weakSamples.map((s) => {
+        const parts = [`### ${s.sample_id}（${s.compositeScore}/5.0）`];
+        if (s.llmReason)
+            parts.push(`评委反馈: ${s.llmReason}`);
+        if (s.failedAssertions.length > 0)
+            parts.push(`失败断言: ${s.failedAssertions.join(', ')}`);
+        if (s.diagnostic) {
+            if (s.diagnostic.rootCause?.length)
+                parts.push(`诊断归因: ${s.diagnostic.rootCause.join(', ')}`);
+            if (s.diagnostic.summary)
+                parts.push(`诊断摘要: ${s.diagnostic.summary}`);
+            if (s.diagnostic.expected)
+                parts.push(`期望行为: ${s.diagnostic.expected}`);
+            if (s.diagnostic.actual)
+                parts.push(`实际行为: ${s.diagnostic.actual}`);
+            if (s.diagnostic.skillSuggestion)
+                parts.push(`建议修改 skill: ${s.diagnostic.skillSuggestion}`);
+            if (s.diagnostic.sampleSuggestion)
+                parts.push(`建议修改 sample: ${s.diagnostic.sampleSuggestion}`);
+            if (s.diagnostic.none)
+                parts.push(`无需改动说明: ${s.diagnostic.none}`);
+        }
+        if (s.dimensions) {
+            const dimStr = Object.entries(s.dimensions).map(([k, v]) => `${k}: ${v}`).join(', ');
+            parts.push(`维度分数: ${dimStr}`);
+        }
+        return parts.join('\n');
+    }).join('\n\n');
+    return `## 当前 Skill（平均分: ${score.toFixed(2)}/5.0）
+
+${skillContent}
+
+## 低分用例分析
+
+${weakDetails || '（无低分用例）'}`;
+}
+function buildImprovementSuffix(mode, candidatePath) {
+    if (mode === 'agent') {
+        return `\n\n请使用 Edit 工具修改文件 ${candidatePath}，只改与上述问题直接相关的部分。不要重写整篇文件。`;
+    }
+    return '\n\n请基于以上分析改进 Skill，使其在这些场景中表现更好。直接输出改进后的完整 Skill 内容。';
+}
+function parseImprovedSkill(output) {
+    let content = output.trim();
+    // Strip markdown code fences if present
+    const match = content.match(/```(?:markdown|md)?\s*([\s\S]*?)```/);
+    if (match)
+        content = match[1].trim();
+    // Strip leading explanation lines before the actual skill content
+    if (content.startsWith('以下是') || content.startsWith('改进后')) {
+        const lines = content.split('\n');
+        const startIdx = lines.findIndex((l) => l.startsWith('#') || l.startsWith('你是'));
+        if (startIdx > 0)
+            content = lines.slice(startIdx).join('\n');
+    }
+    return content;
+}
+export function mergeEvolveReports(roundReports, skillName, totalCostUSD, samples, skillPath) {
+    const firstReport = roundReports[0].report;
+    // Build variant labels: "round-0", "round-1", "round-2", ...
+    const variantLabels = roundReports.map(({ round }) => `round-${round}`);
+    // Build summary: map each variant label to its round's summary
+    const summary = {};
+    for (let i = 0; i < roundReports.length; i++) {
+        const { report } = roundReports[i];
+        const originalKey = Object.keys(report.summary)[0];
+        summary[variantLabels[i]] = report.summary[originalKey];
+    }
+    // Build results: merge per-sample variant data across rounds
+    const sampleIds = firstReport.results.map((r) => r.sample_id);
+    const results = sampleIds.map((sampleId) => {
+        const variants = {};
+        for (let i = 0; i < roundReports.length; i++) {
+            const entry = roundReports[i].report.results.find((r) => r.sample_id === sampleId);
+            if (entry) {
+                const originalKey = Object.keys(entry.variants)[0];
+                variants[variantLabels[i]] = entry.variants[originalKey];
+            }
+        }
+        return { sample_id: sampleId, variants };
+    });
+    // Build variantConfigs: collect from each round, relabel variant name; mark round-0 as baseline
+    const variantConfigs = roundReports.flatMap(({ round, report }, i) => (report.meta.variantConfigs || []).map((cfg) => ({
+        ...cfg,
+        variant: variantLabels[i],
+        ...(round === 0 ? { experimentType: 'baseline', artifactKind: 'baseline', executionStrategy: 'baseline' } : {}),
+    })));
+    // Build artifactHashes: collect from each round, relabel key
+    const artifactHashes = {};
+    for (let i = 0; i < roundReports.length; i++) {
+        const hashes = roundReports[i].report.meta.artifactHashes || {};
+        const originalKey = Object.keys(hashes)[0];
+        if (originalKey)
+            artifactHashes[variantLabels[i]] = hashes[originalKey];
+    }
+    const runId = `evolve-${skillName}-${generateRunId([skillName]).split('-').slice(-2).join('-')}`;
+    const report = {
+        kind: 'evaluation',
+        id: runId,
+        meta: {
+            ...firstReport.meta,
+            variants: variantLabels,
+            variantConfigs,
+            artifactHashes,
+            totalCostUSD: Number(totalCostUSD.toFixed(6)),
+            timestamp: new Date().toISOString(),
+            evolve: {
+                skillName,
+                ...(skillPath ? { skillPath } : {}),
+            },
+        },
+        summary,
+        results,
+        ...(firstReport.sampleSnapshots ? { sampleSnapshots: firstReport.sampleSnapshots } : {}),
+    };
+    // pass samples to populate analysis.sampleQuality (capability/difficulty/
+    // construct/provenance coverage). Without samples, sampleQuality is omitted but
+    // structured insights are still computed.
+    report.analysis = analyzeResults(report, { samples });
+    return report;
+}
+export async function evolveSkill({ skillPath, samplesPath, rounds = 5, target = null, stopOnAssertionsPass = false, autoFixSamples = false, sampleFixMaxAttempts = 2, reuseLatestEval = false, model = DEFAULT_MODEL, judgeModels, improveModel = DEFAULT_MODEL, improveMode = 'agent', executorName = 'claude', concurrency = 1, timeoutMs, skipConnectivity = false, effort, noDiagnostic, skipDoctor, holdoutRatio = 0, onProgress = null, onRoundProgress = null, }) {
+    if (judgeModels && judgeModels.length > 1) {
+        throw new Error('evolveSkill does not support multi-judge ensemble (received '
+            + `${judgeModels.length} judges). Pass a single-judge array, e.g. `
+            + `[{ executor: 'claude', model: 'haiku' }]`);
+    }
+    const effectiveJudgeModels = judgeModels && judgeModels.length > 0
+        ? judgeModels
+        : [{ executor: executorName, model: JUDGE_MODEL }];
+    const absSkillPath = resolve(skillPath);
+    const absSamplesPath = resolve(samplesPath);
+    const skillDir = dirname(absSkillPath);
+    const fileBaseName = basename(absSkillPath, '.md');
+    const skillName = readSkillName(absSkillPath) ?? (fileBaseName === 'SKILL' ? basename(skillDir) : fileBaseName);
+    const evolveDir = join(skillDir, 'evolve');
+    if (!existsSync(absSkillPath))
+        throw new Error(`skill file not found: ${absSkillPath}`);
+    if (!existsSync(absSamplesPath))
+        throw new Error(`samples file not found: ${absSamplesPath}`);
+    mkdirSync(evolveDir, { recursive: true });
+    // Holdout split (opt-in). Computed once over the canonical sample order so it's
+    // stable across rounds. When active, accept decisions use the holdout composite
+    // and weak-sample extraction only sees the training split — the skill is never
+    // tuned to the samples that decide whether it's accepted.
+    const allSampleIds = loadSamples(absSamplesPath).samples.map((s) => s.sample_id);
+    const holdoutSplit = splitHoldout(allSampleIds, holdoutRatio);
+    const holdoutInfo = holdoutRatio > 0
+        ? {
+            ratio: holdoutRatio,
+            trainCount: holdoutSplit?.trainIds.size ?? allSampleIds.length,
+            holdoutCount: holdoutSplit?.holdoutIds.size ?? 0,
+            ...(holdoutSplit ? {} : { disabled: true }),
+        }
+        : undefined;
+    // Accept-decision score for a report's variant: holdout composite when the split
+    // is active, otherwise the full-set composite (legacy behavior).
+    const decisionScore = (report, key) => holdoutSplit
+        ? subsetCompositeScore(report, key, holdoutSplit.holdoutIds)
+        : (report.summary[key]?.avgCompositeScore ?? 0);
+    const trainScoreOf = (report, key) => holdoutSplit ? subsetCompositeScore(report, key, holdoutSplit.trainIds) : undefined;
+    // Per-round trajectory tail: train / holdout breakdown, only when split active.
+    const splitScores = (report, key, decision) => holdoutSplit ? { trainScore: trainScoreOf(report, key), holdoutScore: decision } : {};
+    // Save original as r0
+    let currentBest = readFileSync(absSkillPath, 'utf-8').trim();
+    const r0Path = join(evolveDir, `${skillName}.r0.md`);
+    writeFileSync(r0Path, currentBest);
+    const allVersions = [r0Path];
+    let bestScore = 0;
+    let bestRound = 0;
+    let totalCostUSD = 0;
+    let totalCostReported = true; // 任一 round 走不报 cost 的 executor → 整体 lower-bound
+    let consecutiveRejects = 0;
+    const trajectory = [];
+    const roundReports = [];
+    const sampleFixes = [];
+    let reusedBaselineReportId;
+    let baselineReused = false;
+    let stopReason = 'rounds';
+    // 给定一个 report 看任一 variant 的 exec/judge cost 是否未报告
+    const reportHasUnreportedCost = (rep) => Object.values(rep.summary).some((v) => v.execCostReported === false || v.judgeCostReported === false);
+    // Round 0: baseline evaluation
+    let baselineReport = reuseLatestEval
+        ? await findReusableBaselineReport({
+            skillContent: currentBest,
+            samplesPath: absSamplesPath,
+            model,
+            executorName,
+            judgeModels: effectiveJudgeModels,
+            effort,
+            noDiagnostic,
+        })
+        : null;
+    if (baselineReport) {
+        reusedBaselineReportId = baselineReport.id;
+        baselineReused = true;
+    }
+    else {
+        baselineReport = await evaluate(r0Path, {
+            samplesPath: absSamplesPath, skillDir, model, judgeModels: effectiveJudgeModels, executorName, concurrency, timeoutMs, skipConnectivity, effort, noDiagnostic, skipDoctor, onProgress,
+        });
+    }
+    const baselineVariantKey = Object.keys(baselineReport.summary)[0];
+    bestScore = decisionScore(baselineReport, baselineVariantKey);
+    const baselineCost = baselineReused ? 0 : baselineReport.meta.totalCostUSD;
+    totalCostUSD += baselineCost;
+    const baselineCostReported = baselineReused || !reportHasUnreportedCost(baselineReport);
+    if (!baselineCostReported)
+        totalCostReported = false;
+    trajectory.push({ round: 0, score: bestScore, delta: 0, accepted: true, costUSD: baselineCost, ...splitScores(baselineReport, baselineVariantKey, bestScore) });
+    roundReports.push({ round: 0, accepted: true, report: baselineReport });
+    if (onRoundProgress)
+        onRoundProgress({ round: 0, totalRounds: rounds, phase: 'baseline', score: bestScore, costUSD: baselineCost, costReported: baselineCostReported, reused: baselineReused });
+    if (stopOnAssertionsPass && allNonTripwireAssertionsPass(baselineReport, baselineVariantKey)) {
+        stopReason = 'assertions-pass';
+        writeFileSync(absSkillPath, currentBest);
+        const { samples } = loadSamples(absSamplesPath);
+        const mergedReport = mergeEvolveReports(roundReports, skillName, totalCostUSD, samples, absSkillPath);
+        persistReport(mergedReport, DEFAULT_OUTPUT_DIR);
+        return {
+            startScore: bestScore,
+            finalScore: bestScore,
+            bestRound,
+            totalRounds: 0,
+            totalCostUSD: Number(totalCostUSD.toFixed(6)),
+            stopReason,
+            ...(sampleFixes.length > 0 && { sampleFixes }),
+            ...(reusedBaselineReportId && { reusedBaselineReportId }),
+            ...(totalCostReported ? {} : { costReported: false }),
+            ...(holdoutInfo ? { holdout: holdoutInfo } : {}),
+            trajectory,
+            bestSkillPath: allVersions[bestRound],
+            allVersions,
+            reportId: mergedReport.id,
+        };
+    }
+    // Evolution loop
+    for (let round = 1; round <= rounds; round++) {
+        // Extract weak samples from last accepted evaluation. round=1 reuses baselineReport
+        // (already paid for above). round≥2 re-evaluates the current best skill to get fresh
+        // weak samples — this evaluation costs real money and MUST be added to totalCostUSD,
+        // otherwise the user-facing budget tracker undercounts by ~N-1 evaluations across
+        // an N-round run. (Bug found in  UltraReview.)
+        let lastReport;
+        if (round === 1) {
+            lastReport = baselineReport;
+        }
+        else {
+            lastReport = await evaluate(allVersions[bestRound], {
+                samplesPath: absSamplesPath, skillDir, model, judgeModels: effectiveJudgeModels, executorName, concurrency, timeoutMs, skipConnectivity, effort, noDiagnostic, skipDoctor, onProgress,
+            });
+            totalCostUSD += lastReport.meta.totalCostUSD;
+            if (reportHasUnreportedCost(lastReport))
+                totalCostReported = false;
+        }
+        const lastVariantKey = Object.keys(lastReport.summary)[0];
+        const weakSamples = extractWeakSamples(lastReport, lastVariantKey, 5, holdoutSplit?.trainIds);
+        // Generate improvement
+        const candidatePath = join(evolveDir, `${skillName}.r${round}.md`);
+        const basePrompt = buildImprovementPrompt(currentBest, bestScore, weakSamples);
+        const executor = createExecutor(executorName);
+        let candidateContent;
+        let improveCostUSD;
+        let improveCostReported = true;
+        if (improveMode === 'agent') {
+            writeFileSync(candidatePath, currentBest);
+            const improvePrompt = basePrompt + buildImprovementSuffix('agent', candidatePath);
+            const improveResult = await executor({ model: improveModel, system: IMPROVE_AGENT_SYSTEM_PROMPT, prompt: improvePrompt, cwd: skillDir, timeoutMs });
+            if (!improveResult.ok) {
+                if (onRoundProgress)
+                    onRoundProgress({ round, totalRounds: rounds, phase: 'error', error: improveResult.error });
+                consecutiveRejects++;
+                trajectory.push({ round, score: bestScore, delta: 0, accepted: false, costUSD: improveResult.costUSD });
+                totalCostUSD += improveResult.costUSD;
+                if (improveResult.costReportedByExecutor === false)
+                    totalCostReported = false;
+                if (consecutiveRejects >= 2) {
+                    stopReason = 'consecutive-rejects';
+                    break;
+                }
+                continue;
+            }
+            improveCostUSD = improveResult.costUSD;
+            if (improveResult.costReportedByExecutor === false)
+                improveCostReported = false;
+            candidateContent = readFileSync(candidatePath, 'utf-8');
+        }
+        else {
+            const improvePrompt = basePrompt + buildImprovementSuffix('rewrite');
+            const improveResult = await executor({ model: improveModel, system: IMPROVE_SYSTEM_PROMPT, prompt: improvePrompt, timeoutMs, lean: true });
+            if (!improveResult.ok) {
+                if (onRoundProgress)
+                    onRoundProgress({ round, totalRounds: rounds, phase: 'error', error: improveResult.error });
+                consecutiveRejects++;
+                trajectory.push({ round, score: bestScore, delta: 0, accepted: false, costUSD: improveResult.costUSD });
+                totalCostUSD += improveResult.costUSD;
+                if (improveResult.costReportedByExecutor === false)
+                    totalCostReported = false;
+                if (consecutiveRejects >= 2) {
+                    stopReason = 'consecutive-rejects';
+                    break;
+                }
+                continue;
+            }
+            improveCostUSD = improveResult.costUSD;
+            if (improveResult.costReportedByExecutor === false)
+                improveCostReported = false;
+            candidateContent = parseImprovedSkill(improveResult.output);
+            writeFileSync(candidatePath, candidateContent);
+        }
+        if (!improveCostReported)
+            totalCostReported = false;
+        allVersions.push(candidatePath);
+        let preEvalSampleFixCost = 0;
+        if (autoFixSamples) {
+            const sampleFix = await autoFixSamplesAfterSkillRound({
+                samplesPath: absSamplesPath,
+                skillContent: candidateContent,
+                // Under an active holdout, the sample-fixer may only see training-split samples —
+                // never the holdout samples that drive the accept decision (leak guard).
+                report: holdoutSplit ? restrictReportToSamples(lastReport, holdoutSplit.trainIds) : lastReport,
+                treatmentKey: lastVariantKey,
+                executorName,
+                model: improveModel,
+                maxAttemptsPerSample: sampleFixMaxAttempts,
+                improveMode,
+            });
+            if (sampleFix.fixedCount > 0) {
+                sampleFixes.push({ round, fixedCount: sampleFix.fixedCount, costUSD: sampleFix.costUSD });
+            }
+            preEvalSampleFixCost = sampleFix.costUSD;
+            totalCostUSD += sampleFix.costUSD;
+        }
+        // Evaluate candidate with any sample fixes already applied.
+        const candidateReport = await evaluate(candidatePath, {
+            samplesPath: absSamplesPath, skillDir, model, judgeModels: effectiveJudgeModels, executorName, concurrency, timeoutMs, skipConnectivity, effort, noDiagnostic, skipDoctor, onProgress,
+        });
+        const candidateVariantKey = Object.keys(candidateReport.summary)[0];
+        const candidateScore = decisionScore(candidateReport, candidateVariantKey);
+        const roundCost = improveCostUSD + preEvalSampleFixCost + candidateReport.meta.totalCostUSD;
+        const roundCostReported = improveCostReported && !reportHasUnreportedCost(candidateReport);
+        if (!roundCostReported)
+            totalCostReported = false;
+        totalCostUSD += improveCostUSD + candidateReport.meta.totalCostUSD;
+        const accepted = candidateScore > bestScore;
+        if (accepted) {
+            currentBest = candidateContent;
+            bestScore = candidateScore;
+            bestRound = round;
+            consecutiveRejects = 0;
+        }
+        else {
+            consecutiveRejects++;
+        }
+        if (accepted)
+            roundReports.push({ round, accepted, report: candidateReport });
+        const roundDelta = candidateScore - trajectory[trajectory.length - 1].score;
+        trajectory.push({ round, score: candidateScore, delta: roundDelta, accepted, costUSD: roundCost, ...splitScores(candidateReport, candidateVariantKey, candidateScore) });
+        if (onRoundProgress)
+            onRoundProgress({ round, totalRounds: rounds, phase: 'done', score: candidateScore, delta: roundDelta, accepted, costUSD: roundCost, costReported: roundCostReported });
+        // Early stop
+        if (stopOnAssertionsPass && accepted && allNonTripwireAssertionsPass(candidateReport, candidateVariantKey)) {
+            stopReason = 'assertions-pass';
+            break;
+        }
+        if (target && bestScore >= target) {
+            stopReason = 'target';
+            break;
+        }
+        if (consecutiveRejects >= 2) {
+            stopReason = 'consecutive-rejects';
+            break;
+        }
+    }
+    // Write best version back to original file only if an improvement was accepted
+    if (bestRound > 0) {
+        writeFileSync(absSkillPath, currentBest);
+    }
+    // Merge all round reports into one and persist
+    let reportId;
+    if (roundReports.length > 0) {
+        // load samples once to enable analysis.sampleQuality on the merged report.
+        const { samples } = loadSamples(absSamplesPath);
+        const mergedReport = mergeEvolveReports(roundReports, skillName, totalCostUSD, samples, absSkillPath);
+        persistReport(mergedReport, DEFAULT_OUTPUT_DIR);
+        reportId = mergedReport.id;
+    }
+    return {
+        startScore: trajectory[0].score,
+        finalScore: bestScore,
+        bestRound,
+        totalRounds: trajectory.length - 1, // excluding baseline
+        totalCostUSD: Number(totalCostUSD.toFixed(6)),
+        stopReason,
+        ...(sampleFixes.length > 0 && { sampleFixes }),
+        ...(reusedBaselineReportId && { reusedBaselineReportId }),
+        ...(totalCostReported ? {} : { costReported: false }),
+        ...(holdoutInfo ? { holdout: holdoutInfo } : {}),
+        trajectory,
+        bestSkillPath: allVersions[bestRound],
+        allVersions,
+        reportId,
+    };
+}
+async function evaluate(skillFilePath, { samplesPath, skillDir, model, judgeModels, executorName, concurrency, timeoutMs, skipConnectivity, effort, noDiagnostic, skipDoctor, onProgress }) {
+    const { report } = await runEvaluation({
+        samplesPath,
+        skillDir,
+        // evolve 评测每一轮只跑当前迭代的 skill，没有对照组；标为 treatment。
+        variantSpecs: [{ name: skillFilePath, role: 'treatment', expr: skillFilePath }],
+        model,
+        judgeModels,
+        outputDir: null, // don't persist intermediate reports
+        concurrency,
+        timeoutMs,
+        executorName,
+        skipConnectivity,
+        effort,
+        noDiagnostic,
+        skipDoctor,
+        onProgress,
+    });
+    return report;
+}
+//# sourceMappingURL=evolver.js.map