oh-my-knowledge 0.30.0 → 0.32.0

package/dist/src/analysis/report-diagnostics.js

@@ -1,753 +0,0 @@
-/**
- * Auto-analysis: detect patterns and generate insights from evaluation results.
- */
-import { normalizeCapability } from './sample-diagnostics.js';
-/**
- * Analyze an evaluation report and produce structured insights.
- */
-export function analyzeResults(report, opts = {}) {
-    const insights = [];
-    const variants = report.meta?.variants || [];
-    const results = report.results || [];
-    // sampleQuality aggregate is built from sample metadata only,
-    // independent of result data. Computed even when results.length === 0 or
-    // variants.length < 2 (e.g. dry-run / single-variant analysis).
-    const sampleQuality = opts.samples
-        ? buildSampleQualityAggregate(opts.samples)
-        : undefined;
-    if (results.length === 0 || variants.length < 2) {
-        return { insights, ...(sampleQuality && { sampleQuality }) };
-    }
-    // 1. Low-discrimination assertions
-    detectLowDiscrimination(results, variants, insights);
-    // 2. Uniform scores across variants
-    detectUniformScores(results, variants, insights);
-    // 3. All-pass / all-fail assertions
-    detectAllPassFail(results, variants, insights);
-    // 4. High-cost samples
-    detectHighCost(results, variants, insights);
-    // 5. Efficiency gap (turns & cost)
-    detectEfficiencyGap(report, variants, insights);
-    // 6. Agent tool usage patterns
-    detectToolPatterns(report, variants, insights);
-    // 7. Tooling / permission issues
-    detectToolPermissionIssues(results, variants, insights);
-    // 8. Trace integrity
-    detectTraceIntegrity(report, variants, insights);
-    // 9. Agent assertion discrimination
-    detectAgentAssertionDiscrimination(results, variants, insights);
-    // 10. Suggest --repeat when score variance is high and no repeat data
-    detectNeedRepeat(report, results, variants, insights);
-    return {
-        insights,
-        ...(sampleQuality && { sampleQuality }),
-    };
-}
-/**
- * Build sample design science aggregate from sample metadata.
- *
- * Pure function — no result/score data needed. Reads:
- * - `Sample.capability` (string[], normalized case-insensitive + dash/camel/underscore stripped)
- * - `Sample.difficulty` ('easy' | 'medium' | 'hard')
- * - `Sample.construct` (free-form string)
- * - `Sample.provenance` ('human' | 'llm-generated' | 'production-trace')
- * - `Sample.rubric` (for avgRubricLength)
- *
- * Missing fields are bucketed under the `unspecified` key in the relevant
- * distribution map, so users see "I have N samples without difficulty declared".
- *
- * Persisted on the report so studio can surface coverage gaps. Does NOT
- * participate in grading / judge / verdict. See docs/sample-design-spec.md.
- */
-export function buildSampleQualityAggregate(samples) {
-    const capabilityCoverage = {};
-    const difficultyDistribution = {
-        easy: 0, medium: 0, hard: 0, unspecified: 0,
-    };
-    const constructDistribution = {};
-    const provenanceBreakdown = {};
-    let totalRubricLength = 0;
-    let rubricCount = 0;
-    let withCapability = 0;
-    let withDifficulty = 0;
-    let withConstruct = 0;
-    let withProvenance = 0;
-    for (const sample of samples) {
-        // capability — normalize case + dash/camel/underscore so 'api-selection' / 'apiSelection' / 'API_Selection' merge.
-        if (Array.isArray(sample.capability) && sample.capability.length > 0) {
-            withCapability++;
-            const seen = new Set();
-            for (const rawCap of sample.capability) {
-                if (typeof rawCap !== 'string')
-                    continue;
-                const cap = normalizeCapability(rawCap);
-                if (seen.has(cap))
-                    continue; // 同 sample 内同 capability 重复声明只计 1
-                seen.add(cap);
-                capabilityCoverage[cap] = (capabilityCoverage[cap] || 0) + 1;
-            }
-        }
-        // difficulty
-        if (sample.difficulty) {
-            withDifficulty++;
-            difficultyDistribution[sample.difficulty]++;
-        }
-        else {
-            difficultyDistribution.unspecified++;
-        }
-        // construct (free-form)
-        if (sample.construct) {
-            withConstruct++;
-            constructDistribution[sample.construct] = (constructDistribution[sample.construct] || 0) + 1;
-        }
-        else {
-            constructDistribution.unspecified = (constructDistribution.unspecified || 0) + 1;
-        }
-        // provenance
-        if (sample.provenance) {
-            withProvenance++;
-            provenanceBreakdown[sample.provenance] = (provenanceBreakdown[sample.provenance] || 0) + 1;
-        }
-        else {
-            provenanceBreakdown.unspecified = (provenanceBreakdown.unspecified || 0) + 1;
-        }
-        // rubric length(only counted if present, NaN-safe)
-        if (sample.rubric) {
-            totalRubricLength += sample.rubric.trim().length;
-            rubricCount++;
-        }
-    }
-    return {
-        capabilityCoverage,
-        difficultyDistribution,
-        constructDistribution,
-        provenanceBreakdown,
-        avgRubricLength: rubricCount > 0 ? Math.round(totalRubricLength / rubricCount) : 0,
-        sampleCountWithCapability: withCapability,
-        sampleCountWithDifficulty: withDifficulty,
-        sampleCountWithConstruct: withConstruct,
-        sampleCountWithProvenance: withProvenance,
-    };
-}
-export function generateAnalysisSummary(report, lang = 'zh') {
-    const variants = report.meta?.variants || [];
-    if (variants.length < 2)
-        return undefined;
-    const stats = report.summary || {};
-    // Find control group. experimentRole 是 v0.16 起用户显式声明的 control/treatment
-    // 角色(见 docs/terminology-spec.md 三-4),是判定对照组的唯一来源。
-    // 老 report(v0.15 及更早)variantConfig 里可能缺 experimentRole 字段,
-    // 退化到从 artifactKind === 'baseline' / experimentType 反推——标注为 legacy 路径。
-    const configs = report.meta?.variantConfigs || [];
-    let controlVariants = configs
-        .filter((c) => c.experimentRole === 'control')
-        .map((c) => c.variant);
-    if (controlVariants.length === 0) {
-        // legacy fallback for old reports without experimentRole
-        controlVariants = configs
-            .filter((c) => c.artifactKind === 'baseline' || c.experimentType === 'runtime-context-only' || c.experimentType === 'baseline')
-            .map((c) => c.variant);
-    }
-    const control = controlVariants[0] || variants[0];
-    const test = variants.find((v) => v !== control);
-    if (!test || !control)
-        return undefined;
-    const cs = stats[control];
-    const ts = stats[test];
-    if (!cs || !ts)
-        return undefined;
-    const lines = [];
-    // ── Section 1: Core verdict ──
-    const cScore = cs.avgCompositeScore;
-    const tScore = ts.avgCompositeScore;
-    const scoreDiff = tScore != null && cScore != null ? tScore - cScore : null;
-    if (scoreDiff != null && tScore != null && cScore != null) {
-        const absDiff = Math.abs(scoreDiff);
-        if (absDiff < 0.1) {
-            lines.push(lang === 'zh'
-                ? `【结论】${test} 与 ${control} 综合得分持平（${tScore.toFixed(2)} vs ${cScore.toFixed(2)}），质量无显著差异。`
-                : `【Conclusion】${test} and ${control} are effectively tied on composite score (${tScore.toFixed(2)} vs ${cScore.toFixed(2)}); no clear quality difference.`);
-        }
-        else if (scoreDiff > 0) {
-            const tag = lang === 'zh'
-                ? (absDiff > 0.3 ? '明显领先' : '略优')
-                : (absDiff > 0.3 ? 'clearly ahead' : 'slightly ahead');
-            lines.push(lang === 'zh'
-                ? `【结论】${test} 综合得分 ${tag}（${tScore.toFixed(2)} vs ${cScore.toFixed(2)}，+${scoreDiff.toFixed(2)}）。`
-                : `【Conclusion】${test} is ${tag} on composite score (${tScore.toFixed(2)} vs ${cScore.toFixed(2)}, +${scoreDiff.toFixed(2)}).`);
-        }
-        else {
-            const tag = lang === 'zh'
-                ? (absDiff > 0.3 ? '明显落后' : '略低')
-                : (absDiff > 0.3 ? 'clearly behind' : 'slightly lower');
-            lines.push(lang === 'zh'
-                ? `【结论】${test} 综合得分 ${tag}（${tScore.toFixed(2)} vs ${cScore.toFixed(2)}，${scoreDiff.toFixed(2)}）。`
-                : `【Conclusion】${test} is ${tag} on composite score (${tScore.toFixed(2)} vs ${cScore.toFixed(2)}, ${scoreDiff.toFixed(2)}).`);
-        }
-    }
-    // ── Section 2: Key differentiators with concrete numbers ──
-    const diffs = [];
-    // Quality sub-scores — only mention when there IS a difference
-    const tFact = ts.avgFactScore;
-    const cFact = cs.avgFactScore;
-    if (tFact != null && cFact != null) {
-        if (tFact === cFact) {
-            // Both same — don't mention (e.g. both 5/5 is not interesting)
-        }
-        else if (tFact > cFact) {
-            diffs.push(lang === 'zh'
-                ? `事实性 ${tFact.toFixed(1)} vs ${cFact.toFixed(1)}（↑${(tFact - cFact).toFixed(1)}）`
-                : `Fact score ${tFact.toFixed(1)} vs ${cFact.toFixed(1)} (up ${(tFact - cFact).toFixed(1)})`);
-        }
-        else {
-            diffs.push(lang === 'zh'
-                ? `事实性 ${tFact.toFixed(1)} vs ${cFact.toFixed(1)}（↓${(cFact - tFact).toFixed(1)}）`
-                : `Fact score ${tFact.toFixed(1)} vs ${cFact.toFixed(1)} (down ${(cFact - tFact).toFixed(1)})`);
-        }
-    }
-    const tBehavior = ts.avgBehaviorScore;
-    const cBehavior = cs.avgBehaviorScore;
-    if (tBehavior != null && cBehavior != null && Math.abs(tBehavior - cBehavior) > 0.3) {
-        const dir = tBehavior > cBehavior ? '↑' : '↓';
-        diffs.push(lang === 'zh'
-            ? `行为合规 ${tBehavior.toFixed(1)} vs ${cBehavior.toFixed(1)}（${dir}${Math.abs(tBehavior - cBehavior).toFixed(1)}）`
-            : `Behavior score ${tBehavior.toFixed(1)} vs ${cBehavior.toFixed(1)} (${dir === '↑' ? 'up' : 'down'} ${Math.abs(tBehavior - cBehavior).toFixed(1)})`);
-    }
-    const tJudge = ts.avgJudgeScore;
-    const cJudge = cs.avgJudgeScore;
-    if (tJudge != null && cJudge != null && Math.abs(tJudge - cJudge) >= 0.5) {
-        const dir = tJudge > cJudge ? '↑' : '↓';
-        diffs.push(lang === 'zh'
-            ? `LLM 评价 ${tJudge.toFixed(1)} vs ${cJudge.toFixed(1)}（${dir}${Math.abs(tJudge - cJudge).toFixed(1)}）`
-            : `LLM judge ${tJudge.toFixed(1)} vs ${cJudge.toFixed(1)} (${dir === '↑' ? 'up' : 'down'} ${Math.abs(tJudge - cJudge).toFixed(1)})`);
-    }
-    // Efficiency — with percentages
-    const cTurns = cs.avgNumTurns;
-    const tTurns = ts.avgNumTurns;
-    if (cTurns > 0 && tTurns > 0 && cTurns !== tTurns) {
-        const pct = Math.abs(((tTurns - cTurns) / cTurns) * 100).toFixed(0);
-        if (tTurns < cTurns) {
-            diffs.push(lang === 'zh'
-                ? `轮次 ${tTurns.toFixed(1)} vs ${cTurns.toFixed(1)}（↓${pct}%，路径更高效）`
-                : `Turns ${tTurns.toFixed(1)} vs ${cTurns.toFixed(1)} (down ${pct}%, more efficient path)`);
-        }
-        else {
-            diffs.push(lang === 'zh'
-                ? `轮次 ${tTurns.toFixed(1)} vs ${cTurns.toFixed(1)}（↑${pct}%）`
-                : `Turns ${tTurns.toFixed(1)} vs ${cTurns.toFixed(1)} (up ${pct}%)`);
-        }
-    }
-    // Cost — with percentages
-    const cCost = cs.avgCostPerSample;
-    const tCost = ts.avgCostPerSample;
-    if (cCost > 0 && tCost > 0 && Math.abs(tCost - cCost) / cCost > 0.05) {
-        const pct = Math.abs(((tCost - cCost) / cCost) * 100).toFixed(0);
-        if (tCost < cCost) {
-            diffs.push(lang === 'zh'
-                ? `单用例成本 $${tCost.toFixed(4)} vs $${cCost.toFixed(4)}（↓${pct}%）`
-                : `Cost per sample $${tCost.toFixed(4)} vs $${cCost.toFixed(4)} (down ${pct}%)`);
-        }
-        else {
-            diffs.push(lang === 'zh'
-                ? `单用例成本 $${tCost.toFixed(4)} vs $${cCost.toFixed(4)}（↑${pct}%）`
-                : `Cost per sample $${tCost.toFixed(4)} vs $${cCost.toFixed(4)} (up ${pct}%)`);
-        }
-    }
-    // Duration — with percentages
-    const cDur = cs.avgDurationMs;
-    const tDur = ts.avgDurationMs;
-    if (cDur > 0 && tDur > 0 && Math.abs(tDur - cDur) / cDur > 0.1) {
-        const pct = Math.abs(((tDur - cDur) / cDur) * 100).toFixed(0);
-        const tSec = (tDur / 1000).toFixed(1);
-        const cSec = (cDur / 1000).toFixed(1);
-        if (tDur < cDur) {
-            diffs.push(lang === 'zh'
-                ? `耗时 ${tSec}s vs ${cSec}s（↓${pct}%）`
-                : `Duration ${tSec}s vs ${cSec}s (down ${pct}%)`);
-        }
-        else {
-            diffs.push(lang === 'zh'
-                ? `耗时 ${tSec}s vs ${cSec}s（↑${pct}%）`
-                : `Duration ${tSec}s vs ${cSec}s (up ${pct}%)`);
-        }
-    }
-    // Tool usage difference
-    const tTools = ts.avgToolCalls;
-    const cTools = cs.avgToolCalls;
-    if (tTools != null && cTools != null && Math.abs(tTools - cTools) > 0.5) {
-        diffs.push(lang === 'zh'
-            ? `工具调用 ${tTools.toFixed(1)} vs ${cTools.toFixed(1)} 次`
-            : `Tool calls ${tTools.toFixed(1)} vs ${cTools.toFixed(1)}`);
-    }
-    if (diffs.length > 0) {
-        lines.push(lang === 'zh'
-            ? `【关键差异】${diffs.join('；')}。`
-            : `【Key differences】${diffs.join('; ')}.`);
-    }
-    // ── Section 3: Synthesis — connect the dots ──
-    const synthesis = [];
-    // Quality-cost tradeoff insight
-    if (scoreDiff != null && cCost > 0 && tCost > 0) {
-        const costRatio = (tCost - cCost) / cCost;
-        if (Math.abs(scoreDiff) < 0.1 && costRatio < -0.15) {
-            synthesis.push(lang === 'zh'
-                ? `质量相当但成本显著降低，${test} 是更经济的选择`
-                : `similar quality with materially lower cost; ${test} is the more economical choice`);
-        }
-        else if (scoreDiff > 0.1 && costRatio > 0.15) {
-            synthesis.push(lang === 'zh'
-                ? '质量提升伴随成本上涨，需权衡投入产出比'
-                : 'quality improved, but cost also increased; weigh the return on investment');
-        }
-        else if (scoreDiff > 0.1 && costRatio <= 0) {
-            synthesis.push(lang === 'zh'
-                ? `质量与成本双优，${test} 全面领先`
-                : `${test} leads on both quality and cost`);
-        }
-        else if (scoreDiff < -0.1 && costRatio < -0.15) {
-            synthesis.push(lang === 'zh'
-                ? '成本虽降但质量下滑，需评估质量底线是否可接受'
-                : 'cost decreased, but quality dropped; check whether the quality floor is still acceptable');
-        }
-    }
-    // Tool success rate concern
-    const tToolSuccess = ts.toolSuccessRate;
-    if (tToolSuccess != null && tToolSuccess < 1 && tToolSuccess >= 0.5) {
-        synthesis.push(lang === 'zh'
-            ? `${test} 存在工具调用失败（成功率 ${(tToolSuccess * 100).toFixed(0)}%），可能拉低了得分`
-            : `${test} had tool-call failures (${(tToolSuccess * 100).toFixed(0)}% success), which may have pulled the score down`);
-    }
-    // Variance / significance from --repeat
-    if (report.variance) {
-        const v = report.variance;
-        for (const comp of v.comparisons) {
-            if (comp.a === control && comp.b === test || comp.a === test && comp.b === control) {
-                const es = comp.effectSize;
-                let esText = '';
-                if (es && es.primary !== 'none') {
-                    const primaryVal = es.primary === 'g' ? es.hedgesG : es.cohensD;
-                    const secondaryLabel = es.primary === 'g' ? 'd' : 'g';
-                    const secondaryVal = es.primary === 'g' ? es.cohensD : es.hedgesG;
-                    esText = lang === 'zh'
-                        ? `，效应量 ${es.primary}=${primaryVal.toFixed(2)}（${es.magnitude}，${secondaryLabel}=${secondaryVal.toFixed(2)}）`
-                        : `, effect size ${es.primary}=${primaryVal.toFixed(2)} (${es.magnitude}, ${secondaryLabel}=${secondaryVal.toFixed(2)})`;
-                }
-                if (comp.significant) {
-                    synthesis.push(lang === 'zh'
-                        ? `${v.runs} 轮重复评测显示差异具有统计显著性（t=${comp.tStatistic.toFixed(2)}, df=${comp.df.toFixed(1)}, p<0.05${esText}）`
-                        : `${v.runs} repeated runs show a statistically significant difference (t=${comp.tStatistic.toFixed(2)}, df=${comp.df.toFixed(1)}, p<0.05${esText})`);
-                }
-                else {
-                    synthesis.push(lang === 'zh'
-                        ? `${v.runs} 轮重复评测未达到统计显著性（t=${comp.tStatistic.toFixed(2)}, df=${comp.df.toFixed(1)}${esText}），差异可能源于随机波动`
-                        : `${v.runs} repeated runs did not reach statistical significance (t=${comp.tStatistic.toFixed(2)}, df=${comp.df.toFixed(1)}${esText}); the gap may be random variation`);
-                }
-            }
-        }
-        const testVd = v.perVariant[test];
-        if (testVd) {
-            synthesis.push(lang === 'zh'
-                ? `${test} 跨轮 95% 置信区间 [${testVd.lower.toFixed(2)}, ${testVd.upper.toFixed(2)}]`
-                : `${test} cross-run 95% confidence interval [${testVd.lower.toFixed(2)}, ${testVd.upper.toFixed(2)}]`);
-        }
-    }
-    if (synthesis.length > 0) {
-        lines.push(lang === 'zh'
-            ? `【综合洞察】${synthesis.join('；')}。`
-            : `【Synthesis】${synthesis.join('; ')}.`);
-    }
-    // Caveats and recommendations are handled by the issues table below,
-    // so the summary focuses only on verdict + differentiators + synthesis.
-    if (lines.length === 0)
-        return undefined;
-    return lines.join('\n');
-}
-const AGENT_ASSERTION_TYPES = new Set([
-    'tools_called',
-    'tools_not_called',
-    'tool_input_contains',
-    'tool_input_not_contains',
-    'tool_output_contains',
-    'tools_count_min',
-    'tools_count_max',
-    'turns_min',
-    'turns_max',
-]);
-const TRACE_HEAVY_AGENT_ASSERTION_TYPES = new Set([
-    'tools_called',
-    'tools_not_called',
-    'tool_input_contains',
-    'tool_input_not_contains',
-    'tool_output_contains',
-    'tools_count_min',
-    'tools_count_max',
-]);
-function collectAgentAssertionTypes(results, variants) {
-    const types = new Set();
-    for (const result of results) {
-        const details = result.variants?.[variants[0]]?.assertions?.details || [];
-        for (const detail of details) {
-            if (AGENT_ASSERTION_TYPES.has(detail.type)) {
-                types.add(detail.type);
-            }
-        }
-    }
-    return types;
-}
-function detectLowDiscrimination(results, variants, insights) {
-    // For each sample, check if all variants have the same assertion pass/fail pattern
-    const allPassedPatterns = [];
-    const allFailedPatterns = [];
-    for (const r of results) {
-        const firstVariantData = r.variants?.[variants[0]];
-        if (!firstVariantData?.assertions?.details)
-            continue;
-        for (let ai = 0; ai < firstVariantData.assertions.details.length; ai++) {
-            const assertion = firstVariantData.assertions.details[ai];
-            let allSame = true;
-            for (let vi = 1; vi < variants.length; vi++) {
-                const otherData = r.variants?.[variants[vi]];
-                const otherDetail = otherData?.assertions?.details?.[ai];
-                if (!otherDetail || otherDetail.passed !== assertion.passed) {
-                    allSame = false;
-                    break;
-                }
-            }
-            if (allSame && variants.length >= 2) {
-                const entry = {
-                    sample_id: r.sample_id,
-                    type: assertion.type,
-                    value: assertion.value,
-                    allPassed: assertion.passed,
-                };
-                if (assertion.passed) {
-                    allPassedPatterns.push(entry);
-                }
-                else {
-                    allFailedPatterns.push(entry);
-                }
-            }
-        }
-    }
-    if (allPassedPatterns.length > 0) {
-        insights.push({
-            type: 'low_discrimination_all_passed',
-            severity: 'info',
-            details: allPassedPatterns,
-        });
-    }
-    if (allFailedPatterns.length > 0) {
-        insights.push({
-            type: 'low_discrimination_all_failed',
-            severity: 'warning',
-            details: allFailedPatterns,
-        });
-    }
-}
-function detectUniformScores(results, variants, insights) {
-    let uniformCount = 0;
-    const uniformSamples = [];
-    for (const r of results) {
-        const scores = variants
-            .map((v) => r.variants?.[v]?.compositeScore)
-            .filter((s) => typeof s === 'number' && s > 0);
-        if (scores.length >= 2) {
-            const max = Math.max(...scores);
-            const min = Math.min(...scores);
-            if (max - min < 0.5) {
-                uniformCount++;
-                uniformSamples.push(r.sample_id);
-            }
-        }
-    }
-    if (uniformCount > 0) {
-        insights.push({
-            type: 'uniform_scores',
-            severity: uniformCount === results.length ? 'warning' : 'info',
-            details: uniformSamples,
-        });
-    }
-}
-function detectAllPassFail(results, variants, insights) {
-    let allPassCount = 0;
-    let allFailCount = 0;
-    for (const r of results) {
-        for (const v of variants) {
-            const assertions = r.variants?.[v]?.assertions;
-            if (!assertions || assertions.total === 0)
-                continue;
-            if (assertions.passed === assertions.total)
-                allPassCount++;
-            if (assertions.passed === 0)
-                allFailCount++;
-        }
-    }
-    const totalEntries = results.length * variants.length;
-    if (allPassCount === totalEntries && totalEntries > 0) {
-        insights.push({
-            type: 'all_pass',
-            severity: 'warning',
-            details: { allPassCount },
-        });
-    }
-    if (allFailCount === totalEntries && totalEntries > 0) {
-        insights.push({
-            type: 'all_fail',
-            severity: 'error',
-            details: { allFailCount },
-        });
-    }
-}
-function detectNeedRepeat(report, results, variants, insights) {
-    // Skip if already has variance data (i.e. --repeat was used)
-    if (report.variance)
-        return;
-    // Check if any variant has high score spread (max - min >= 2)
-    for (const v of variants) {
-        const s = report.summary?.[v];
-        if (!s || s.minCompositeScore == null || s.maxCompositeScore == null)
-            continue;
-        const spread = s.maxCompositeScore - s.minCompositeScore;
-        if (spread >= 2) {
-            insights.push({
-                type: 'suggest_repeat',
-                severity: 'info',
-                details: { variant: v, min: s.minCompositeScore, max: s.maxCompositeScore, spread },
-            });
-            return; // Only suggest once
-        }
-    }
-}
-function detectEfficiencyGap(report, variants, insights) {
-    if (variants.length < 2)
-        return;
-    const summary = report.summary || {};
-    // Compare first variant (usually baseline) with others
-    const base = summary[variants[0]];
-    if (!base)
-        return;
-    for (let i = 1; i < variants.length; i++) {
-        const other = summary[variants[i]];
-        if (!other)
-            continue;
-        const details = [];
-        // Turns comparison
-        const baseTurns = base.avgNumTurns;
-        const otherTurns = other.avgNumTurns;
-        if (baseTurns > 0 && otherTurns > 0 && baseTurns !== otherTurns) {
-            const turnsDiff = baseTurns - otherTurns;
-            const turnsPct = ((turnsDiff / baseTurns) * 100).toFixed(0);
-            if (Math.abs(turnsDiff / baseTurns) > 0.3) {
-                details.push(turnsDiff > 0
-                    ? `${variants[i]} 平均减少 ${turnsDiff.toFixed(1)} 轮对话（↓${Math.abs(Number(turnsPct))}%）`
-                    : `${variants[i]} 平均增加 ${Math.abs(turnsDiff).toFixed(1)} 轮对话（↑${Math.abs(Number(turnsPct))}%）`);
-            }
-        }
-        // Cost comparison
-        const baseCost = base.avgCostPerSample;
-        const otherCost = other.avgCostPerSample;
-        if (baseCost > 0 && otherCost > 0 && baseCost !== otherCost) {
-            const costDiff = baseCost - otherCost;
-            const costPct = ((costDiff / baseCost) * 100).toFixed(0);
-            if (Math.abs(costDiff / baseCost) > 0.2) {
-                details.push(costDiff > 0
-                    ? `单用例成本降低 ${Math.abs(Number(costPct))}%（$${otherCost.toFixed(4)} vs $${baseCost.toFixed(4)}）`
-                    : `单用例成本增加 ${Math.abs(Number(costPct))}%（$${otherCost.toFixed(4)} vs $${baseCost.toFixed(4)}）`);
-            }
-        }
-        if (details.length > 0) {
-            insights.push({
-                type: 'efficiency_gap',
-                severity: 'info',
-                details: { baseline: variants[0], variant: variants[i], baseTurns, otherTurns, baseCost, otherCost },
-            });
-        }
-    }
-}
-function detectToolPatterns(report, variants, insights) {
-    const summary = report.summary || {};
-    const hasTools = variants.some((v) => summary[v]?.avgToolCalls != null && summary[v].avgToolCalls > 0);
-    if (!hasTools)
-        return;
-    for (let i = 0; i < variants.length; i++) {
-        const v = variants[i];
-        const s = summary[v];
-        if (!s)
-            continue;
-        // Low tool success rate
-        if (s.toolSuccessRate != null && s.toolSuccessRate < 0.8 && s.avgToolCalls != null && s.avgToolCalls > 0) {
-            insights.push({
-                type: 'low_tool_success_rate',
-                severity: 'warning',
-                details: { variant: v, toolSuccessRate: s.toolSuccessRate, avgToolCalls: s.avgToolCalls },
-            });
-        }
-    }
-    // Compare tool counts between variants
-    if (variants.length >= 2) {
-        const base = summary[variants[0]];
-        for (let i = 1; i < variants.length; i++) {
-            const other = summary[variants[i]];
-            if (!base?.avgToolCalls || !other?.avgToolCalls)
-                continue;
-            const diff = other.avgToolCalls - base.avgToolCalls;
-            if (Math.abs(diff) > 2) {
-                insights.push({
-                    type: 'tool_count_gap',
-                    severity: 'info',
-                    details: { baseline: variants[0], variant: variants[i], baseTools: base.avgToolCalls, otherTools: other.avgToolCalls },
-                });
-            }
-        }
-    }
-}
-function detectToolPermissionIssues(results, variants, insights) {
-    const permissionErrors = [];
-    for (const result of results) {
-        for (const variant of variants) {
-            const calls = result.variants?.[variant]?.toolCalls || [];
-            for (const call of calls) {
-                if (call.success)
-                    continue;
-                const output = String(call.output || '');
-                if (/EACCES|permission denied/i.test(output)) {
-                    permissionErrors.push({
-                        variant,
-                        tool: call.tool,
-                        sample_id: result.sample_id,
-                        output: output.slice(0, 200),
-                    });
-                }
-            }
-        }
-    }
-    if (permissionErrors.length === 0)
-        return;
-    insights.push({
-        type: 'tool_permission_error',
-        severity: 'warning',
-        details: permissionErrors.slice(0, 10),
-    });
-}
-function detectTraceIntegrity(report, variants, insights) {
-    const summary = report.summary || {};
-    const agentAssertionTypes = collectAgentAssertionTypes(report.results || [], variants);
-    const needsTraceHeavyCoverage = [...agentAssertionTypes].some((type) => TRACE_HEAVY_AGENT_ASSERTION_TYPES.has(type));
-    const hasAgentLikeData = variants.some((variant) => {
-        const stats = summary[variant];
-        return Boolean(stats?.avgToolCalls || stats?.avgNumTurns > 1);
-    });
-    if (!hasAgentLikeData || !needsTraceHeavyCoverage)
-        return;
-    const weakCoverage = variants
-        .map((variant) => ({
-        variant,
-        traceCoverageRate: summary[variant]?.traceCoverageRate ?? 0,
-        avgAssistantTurns: summary[variant]?.avgAssistantTurns ?? 0,
-        avgToolTurns: summary[variant]?.avgToolTurns ?? 0,
-        avgToolFailures: summary[variant]?.avgToolFailures ?? 0,
-    }))
-        .filter((item) => item.traceCoverageRate < 0.75);
-    if (weakCoverage.length > 0) {
-        insights.push({
-            type: 'trace_integrity_gap',
-            severity: 'warning',
-            details: weakCoverage,
-        });
-    }
-}
-function detectAgentAssertionDiscrimination(results, variants, insights) {
-    const assertionTypes = collectAgentAssertionTypes(results, variants);
-    const hasTraceHeavyAssertions = [...assertionTypes].some((type) => TRACE_HEAVY_AGENT_ASSERTION_TYPES.has(type));
-    if (!hasTraceHeavyAssertions)
-        return;
-    const evaluated = [];
-    let discriminative = 0;
-    let allPass = 0;
-    let allFail = 0;
-    for (const result of results) {
-        const firstDetails = result.variants?.[variants[0]]?.assertions?.details;
-        if (!firstDetails)
-            continue;
-        for (let index = 0; index < firstDetails.length; index++) {
-            const first = firstDetails[index];
-            if (!AGENT_ASSERTION_TYPES.has(first.type))
-                continue;
-            const states = [];
-            for (const variant of variants) {
-                const detail = result.variants?.[variant]?.assertions?.details?.[index];
-                if (!detail)
-                    continue;
-                states.push(detail.passed);
-            }
-            if (states.length < 2)
-                continue;
-            const pattern = states.map((state) => (state ? 'T' : 'F')).join('/');
-            evaluated.push({
-                sample_id: result.sample_id,
-                type: first.type,
-                value: first.value,
-                pattern,
-            });
-            const unique = new Set(states);
-            if (unique.size > 1)
-                discriminative++;
-            else if (states.every(Boolean))
-                allPass++;
-            else
-                allFail++;
-        }
-    }
-    if (evaluated.length < 4)
-        return;
-    const discriminationRate = Number((discriminative / evaluated.length).toFixed(2));
-    if (discriminationRate < 0.3) {
-        insights.push({
-            type: 'agent_assertion_discrimination_low',
-            severity: 'warning',
-            details: {
-                total: evaluated.length,
-                discriminative,
-                allPass,
-                allFail,
-                examples: evaluated.slice(0, 10),
-            },
-        });
-    }
-    else {
-        insights.push({
-            type: 'agent_assertion_discrimination_ok',
-            severity: 'info',
-            details: {
-                total: evaluated.length,
-                discriminative,
-                allPass,
-                allFail,
-            },
-        });
-    }
-}
-function detectHighCost(results, variants, insights) {
-    const costs = [];
-    for (const r of results) {
-        let sampleCost = 0;
-        for (const v of variants) {
-            sampleCost += r.variants?.[v]?.costUSD || 0;
-        }
-        costs.push({ sample_id: r.sample_id, costUSD: sampleCost });
-    }
-    if (costs.length < 2)
-        return;
-    const avg = costs.reduce((s, c) => s + c.costUSD, 0) / costs.length;
-    if (avg === 0)
-        return;
-    const expensive = costs.filter((c) => c.costUSD > avg * 2);
-    if (expensive.length > 0) {
-        insights.push({
-            type: 'high_cost_sample',
-            severity: 'info',
-            details: expensive,
-        });
-    }
-}
-//# sourceMappingURL=report-diagnostics.js.map