oh-my-knowledge 0.18.0 → 0.20.0

package/dist/src/analysis/sample-diagnostics.js

@@ -0,0 +1,259 @@
+/**
+ * Per-sample quality diagnostics.
+ *
+ * Why this exists
+ * ---------------
+ * `report-diagnostics.ts` answers "what's wrong with the report overall" —
+ * aggregate-level insights about assertions and variant comparisons. This
+ * module answers a complementary question: "which specific samples in my
+ * eval-samples set are problematic and should be fixed before I trust any
+ * conclusion drawn on this report?"
+ *
+ * Senior-engineer pain point: 100 eval-samples typically have 20-40 that
+ * (a) all variants get the same score on (no discriminative power), or
+ * (b) are near-duplicates of another sample (data redundancy inflating N), or
+ * (c) have ambiguous rubric (judge gives wildly different scores on repeats), or
+ * (d) are cost / latency outliers (one sample spikes the budget by itself), or
+ * (e) fail across the board (likely broken sample, not a model deficiency).
+ *
+ * The CLI `omk bench diagnose <reportId>` surfaces these so users can iterate
+ * the SAMPLE SET as much as they iterate the SKILL.
+ *
+ * Output shape: a flat list of issues, each tagged with severity and the
+ * minimal evidence needed to act ("here are the matching prompts"). Issues are
+ * sorted by severity then by sample_id so the report is stable across runs.
+ */
+import { rougeN } from '../grading/assertions.js';
+const DEFAULTS = {
+    duplicateRouge: 0.7,
+    ambiguousStddev: 1.0,
+    costOutlierK: 3,
+    latencyOutlierK: 3,
+    flatThreshold: 0.5,
+};
+const SEVERITY_RANK = { error: 0, warning: 1, info: 2 };
+export function diagnoseSamples(report, options = {}) {
+    const opt = { ...DEFAULTS, ...options };
+    const variants = report.meta?.variants ?? [];
+    const results = report.results ?? [];
+    const issues = [];
+    if (results.length === 0) {
+        return emptyReport(0);
+    }
+    // Pass 1: per-sample score & cost statistics, plus issue detection that
+    // depends only on a single sample's data.
+    const sampleStats = [];
+    for (const entry of results) {
+        const scores = [];
+        let cost = 0;
+        let latencyMs = 0;
+        let errors = 0;
+        const judgeStddevs = [];
+        for (const v of variants) {
+            const r = entry.variants?.[v];
+            if (!r)
+                continue;
+            if (typeof r.compositeScore === 'number' && r.compositeScore > 0)
+                scores.push(r.compositeScore);
+            cost += r.costUSD ?? 0;
+            latencyMs += (r.timing?.totalMs ?? r.durationMs ?? 0);
+            if (r.ok === false)
+                errors++;
+            if (typeof r.llmScoreStddev === 'number' && r.llmScoreStddev > 0)
+                judgeStddevs.push(r.llmScoreStddev);
+        }
+        sampleStats.push({ entry, scores, cost, latencyMs, errors, judgeStddevs });
+        // Flat / all-pass / all-fail (within-sample score spread).
+        if (scores.length >= 2) {
+            const max = Math.max(...scores);
+            const min = Math.min(...scores);
+            if (max === 5 && min === 5) {
+                issues.push({
+                    sample_id: entry.sample_id, severity: 'info', kind: 'all_pass',
+                    message: `所有 variant 得分均为 5 — 样本可能太简单或断言过宽`,
+                    evidence: { scores: scoresMap(entry, variants) },
+                });
+            }
+            else if (max === 1 && min === 1) {
+                issues.push({
+                    sample_id: entry.sample_id, severity: 'error', kind: 'all_fail',
+                    message: `所有 variant 得分均为 1 — 样本可能 broken / rubric 不可达`,
+                    evidence: { scores: scoresMap(entry, variants) },
+                });
+            }
+            else if (max - min < opt.flatThreshold) {
+                issues.push({
+                    sample_id: entry.sample_id, severity: 'warning', kind: 'flat_scores',
+                    message: `分差 ${(max - min).toFixed(2)} < ${opt.flatThreshold} — 区分度低,该样本对结论贡献小`,
+                    evidence: { scores: scoresMap(entry, variants), spread: Number((max - min).toFixed(2)) },
+                });
+            }
+        }
+        // Errored on at least one variant — sample may be broken (env / executor / fixture).
+        if (errors > 0) {
+            issues.push({
+                sample_id: entry.sample_id, severity: errors === variants.length ? 'error' : 'warning', kind: 'error_prone',
+                message: `${errors}/${variants.length} variant 执行失败 — 检查样本配置 / 环境依赖`,
+                evidence: { errorCount: errors, variantCount: variants.length },
+            });
+        }
+        // Ambiguous rubric — high judge stddev (only available when judgeRepeat > 1).
+        const maxStddev = judgeStddevs.length > 0 ? Math.max(...judgeStddevs) : 0;
+        if (maxStddev >= opt.ambiguousStddev) {
+            issues.push({
+                sample_id: entry.sample_id, severity: 'warning', kind: 'ambiguous_rubric',
+                message: `LLM 评委多次评分 stddev ${maxStddev.toFixed(2)} ≥ ${opt.ambiguousStddev} — rubric 可能存在歧义`,
+                evidence: { maxStddev: Number(maxStddev.toFixed(2)), stddevs: judgeStddevs.map((s) => Number(s.toFixed(2))) },
+            });
+        }
+    }
+    // Pass 2: cost / latency outliers via median-multiplier.
+    const costs = sampleStats.map((s) => s.cost).filter((c) => c > 0).sort((a, b) => a - b);
+    const latencies = sampleStats.map((s) => s.latencyMs).filter((l) => l > 0).sort((a, b) => a - b);
+    const medianCost = costs.length > 0 ? costs[Math.floor(costs.length / 2)] : 0;
+    const medianLatency = latencies.length > 0 ? latencies[Math.floor(latencies.length / 2)] : 0;
+    for (const s of sampleStats) {
+        if (medianCost > 0 && s.cost >= opt.costOutlierK * medianCost) {
+            issues.push({
+                sample_id: s.entry.sample_id, severity: 'info', kind: 'cost_outlier',
+                message: `样本总成本 $${s.cost.toFixed(4)} ≥ ${opt.costOutlierK}× 中位数 $${medianCost.toFixed(4)}`,
+                evidence: { cost: Number(s.cost.toFixed(4)), medianCost: Number(medianCost.toFixed(4)) },
+            });
+        }
+        if (medianLatency > 0 && s.latencyMs >= opt.latencyOutlierK * medianLatency) {
+            issues.push({
+                sample_id: s.entry.sample_id, severity: 'info', kind: 'latency_outlier',
+                message: `样本总耗时 ${(s.latencyMs / 1000).toFixed(1)}s ≥ ${opt.latencyOutlierK}× 中位数 ${(medianLatency / 1000).toFixed(1)}s`,
+                evidence: { latencyMs: s.latencyMs, medianMs: medianLatency },
+            });
+        }
+    }
+    // Pass 3: near-duplicate prompts. Requires the caller to pass `samples`
+    // (the original Sample[] from the eval-samples file) — the Report alone
+    // doesn't carry prompts. CLI handler reads samples on the fly. To avoid
+    // O(N²) blowup on huge sets we cap at 500 samples.
+    if (options.samples && options.samples.length > 0 && options.samples.length <= 500) {
+        const samplesById = new Map();
+        for (const s of options.samples)
+            samplesById.set(s.sample_id, s);
+        const prompts = [];
+        for (const entry of results) {
+            const sample = samplesById.get(entry.sample_id);
+            if (sample?.prompt)
+                prompts.push({ id: entry.sample_id, text: sample.prompt });
+        }
+        const seenPair = new Set();
+        for (let i = 0; i < prompts.length; i++) {
+            for (let j = i + 1; j < prompts.length; j++) {
+                const score = rougeN(prompts[i].text, prompts[j].text, 1);
+                if (score >= opt.duplicateRouge) {
+                    const key = [prompts[i].id, prompts[j].id].sort().join('|');
+                    if (seenPair.has(key))
+                        continue;
+                    seenPair.add(key);
+                    issues.push({
+                        sample_id: prompts[i].id, severity: 'warning', kind: 'near_duplicate',
+                        message: `prompt ROUGE-1 ${score.toFixed(2)} ≥ ${opt.duplicateRouge} 与样本 "${prompts[j].id}" 高度相似`,
+                        evidence: { duplicateOf: prompts[j].id, rouge1: Number(score.toFixed(2)) },
+                    });
+                }
+            }
+        }
+    }
+    // Sort and roll up.
+    issues.sort((a, b) => SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity] || a.sample_id.localeCompare(b.sample_id));
+    const byKind = {};
+    for (const i of issues) {
+        if (!byKind[i.kind])
+            byKind[i.kind] = [];
+        if (!byKind[i.kind].includes(i.sample_id))
+            byKind[i.kind].push(i.sample_id);
+    }
+    const totals = {
+        samples: results.length,
+        flagged: new Set(issues.map((i) => i.sample_id)).size,
+        errors: issues.filter((i) => i.severity === 'error').length,
+        warnings: issues.filter((i) => i.severity === 'warning').length,
+        infos: issues.filter((i) => i.severity === 'info').length,
+    };
+    // Health score: 100 minus weighted issue counts, normalized to N samples.
+    // Each error costs 8 points, warning 3, info 1 — capped at 100 deduction.
+    const rawDeduction = totals.errors * 8 + totals.warnings * 3 + totals.infos * 1;
+    const normalized = results.length > 0 ? rawDeduction / results.length : 0;
+    // Map to 0-100. Most healthy: deduction near 0 → score ~100.
+    // Heavily flagged: deduction > 5 per sample → score → 0.
+    const healthScore = Math.max(0, Math.min(100, Math.round(100 - normalized * 20)));
+    return { issues, healthScore, byKind, totals };
+}
+function emptyReport(samples) {
+    return {
+        issues: [],
+        healthScore: 100,
+        byKind: {},
+        totals: { samples, flagged: 0, errors: 0, warnings: 0, infos: 0 },
+    };
+}
+function scoresMap(entry, variants) {
+    const out = {};
+    for (const v of variants) {
+        const r = entry.variants?.[v];
+        if (r && typeof r.compositeScore === 'number') {
+            out[v] = Number(r.compositeScore.toFixed(2));
+        }
+    }
+    return out;
+}
+/**
+ * Plain-text formatter for `omk bench diagnose <reportId>`.
+ */
+export function formatSampleDiagnostics(diag, options = {}) {
+    const lines = [];
+    const { topN } = options;
+    lines.push('');
+    lines.push(`  样本质量诊断 — health score ${diag.healthScore}/100`);
+    lines.push(`  样本总数: ${diag.totals.samples}, flagged: ${diag.totals.flagged} (errors=${diag.totals.errors}, warnings=${diag.totals.warnings}, infos=${diag.totals.infos})`);
+    lines.push('');
+    if (diag.issues.length === 0) {
+        lines.push('  ✓ 未检测到样本质量问题');
+        lines.push('');
+        return lines.join('\n');
+    }
+    // Group by kind for readability.
+    const kinds = Object.keys(diag.byKind);
+    for (const kind of kinds) {
+        const ids = diag.byKind[kind] ?? [];
+        if (ids.length === 0)
+            continue;
+        lines.push(`  [${kind}] ${ids.length} sample(s)`);
+        const matching = diag.issues.filter((i) => i.kind === kind);
+        const display = topN ? matching.slice(0, topN) : matching;
+        for (const issue of display) {
+            const sev = issue.severity === 'error' ? '✗' : issue.severity === 'warning' ? '⚠' : 'ℹ';
+            lines.push(`    ${sev} ${issue.sample_id}: ${issue.message}`);
+        }
+        if (topN && matching.length > topN) {
+            lines.push(`    ... 还有 ${matching.length - topN} 个,加 --top 0 看全部`);
+        }
+        lines.push('');
+    }
+    // Recommendations based on dominant issue kinds.
+    lines.push('  建议:');
+    if (diag.byKind.all_pass && diag.byKind.all_pass.length > diag.totals.samples * 0.3) {
+        lines.push('  - 样本太简单 (>30% all-pass) — 加难度 / 加更严断言来拉开 variant 差距');
+    }
+    if (diag.byKind.flat_scores && diag.byKind.flat_scores.length > diag.totals.samples * 0.3) {
+        lines.push('  - 多数样本区分度低 — 当前评分维度可能与 skill 差异不正交');
+    }
+    if (diag.byKind.near_duplicate) {
+        lines.push('  - 删除或改写 near-duplicate 样本,避免数据有效维度被挤压');
+    }
+    if (diag.byKind.ambiguous_rubric) {
+        lines.push('  - rubric 在这些样本上歧义大 — 改写 rubric 或加更多确定性断言锚定');
+    }
+    if (diag.byKind.error_prone) {
+        lines.push('  - 这些样本执行失败 — 检查环境依赖 / executor 配置 / 样本本身是否过期');
+    }
+    lines.push('');
+    return lines.join('\n');
+}
+//# sourceMappingURL=sample-diagnostics.js.map

package/dist/src/analysis/sample-diagnostics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sample-diagnostics.js","sourceRoot":"","sources":["../../../src/analysis/sample-diagnostics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAGH,OAAO,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAC;AAoDlD,MAAM,QAAQ,GAA+C;IAC3D,cAAc,EAAE,GAAG;IACnB,eAAe,EAAE,GAAG;IACpB,YAAY,EAAE,CAAC;IACf,eAAe,EAAE,CAAC;IAClB,aAAa,EAAE,GAAG;CACnB,CAAC;AAEF,MAAM,aAAa,GAA4C,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;AAEjG,MAAM,UAAU,eAAe,CAAC,MAAc,EAAE,UAA2B,EAAE;IAC3E,MAAM,GAAG,GAAG,EAAE,GAAG,QAAQ,EAAE,GAAG,OAAO,EAAE,CAAC;IACxC,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,EAAE,QAAQ,IAAI,EAAE,CAAC;IAC7C,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;IAErC,MAAM,MAAM,GAAkB,EAAE,CAAC;IAEjC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,WAAW,CAAC,CAAC,CAAC,CAAC;IACxB,CAAC;IAED,wEAAwE;IACxE,0CAA0C;IAC1C,MAAM,WAAW,GAOZ,EAAE,CAAC;IAER,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,MAAM,CAAC,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;YAC9B,IAAI,CAAC,CAAC;gBAAE,SAAS;YACjB,IAAI,OAAO,CAAC,CAAC,cAAc,KAAK,QAAQ,IAAI,CAAC,CAAC,cAAc,GAAG,CAAC;gBAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;YAChG,IAAI,IAAI,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC;YACvB,SAAS,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,OAAO,IAAI,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC;YACtD,IAAI,CAAC,CAAC,EAAE,KAAK,KAAK;gBAAE,MAAM,EAAE,CAAC;YAC7B,IAAI,OAAO,CAAC,CAAC,cAAc,KAAK,QAAQ,IAAI,CAAC,CAAC,cAAc,GAAG,CAAC;gBAAE,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;QACxG,CAAC;QACD,WAAW,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;QAE3E,2DAA2D;QAC3D,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACvB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;YAChC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;YAChC,IAAI,GAAG,KAAK,CAAC,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC;gBAC3B,MAAM,CAAC,IAAI,CAAC;oBACV,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU;oBAC9D,OAAO,EAAE,kCAAkC;oBAC3C,QAAQ,EAAE,EAAE,MAAM,EAAE,SAAS,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE;iBACjD,CAAC,CAAC;YACL,CAAC;iBAAM,IAAI,GAAG,KAAK,CAAC,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC;gBAClC,MAAM,CAAC,IAAI,CAAC;oBACV,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU;oBAC/D,OAAO,EAAE,8CAA8C;oBACvD,QAAQ,EAAE,EAAE,MAAM,EAAE,SAAS,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE;iBACjD,CAAC,CAAC;YACL,CAAC;iBAAM,IAAI,GAAG,GAAG,GAAG,GAAG,GAAG,CAAC,aAAa,EAAE,CAAC;gBACzC,MAAM,CAAC,IAAI,CAAC;oBACV,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,aAAa;oBACpE,OAAO,EAAE,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,aAAa,mBAAmB;oBAC/E,QAAQ,EAAE,EAAE,MAAM,EAAE,SAAS,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE;iBACzF,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,qFAAqF;QACrF,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC;gBACV,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,MAAM,KAAK,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,EAAE,IAAI,EAAE,aAAa;gBAC3G,OAAO,EAAE,GAAG,MAAM,IAAI,QAAQ,CAAC,MAAM,+BAA+B;gBACpE,QAAQ,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,MAAM,EAAE;aAChE,CAAC,CAAC;QACL,CAAC;QAED,8EAA8E;QAC9E,MAAM,SAAS,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1E,IAAI,SAAS,IAAI,GAAG,CAAC,eAAe,EAAE,CAAC;YACrC,MAAM,CAAC,IAAI,CAAC;gBACV,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,kBAAkB;gBACzE,OAAO,EAAE,qBAAqB,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,eAAe,kBAAkB;gBAC7F,QAAQ,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE;aAC9G,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,yDAAyD;IACzD,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACxF,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACjG,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9E,MAAM,aAAa,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAE7F,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;QAC5B,IAAI,UAAU,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC,YAAY,GAAG,UAAU,EAAE,CAAC;YAC9D,MAAM,CAAC,IAAI,CAAC;gBACV,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc;gBACpE,OAAO,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,YAAY,UAAU,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;gBAC3F,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE;aACzF,CAAC,CAAC;QACL,CAAC;QACD,IAAI,aAAa,GAAG,CAAC,IAAI,CAAC,CAAC,SAAS,IAAI,GAAG,CAAC,eAAe,GAAG,aAAa,EAAE,CAAC;YAC5E,MAAM,CAAC,IAAI,CAAC;gBACV,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,iBAAiB;gBACvE,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,eAAe,SAAS,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;gBACxH,QAAQ,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,QAAQ,EAAE,aAAa,EAAE;aAC9D,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,wEAAwE;IACxE,wEAAwE;IACxE,wEAAwE;IACxE,mDAAmD;IACnD,IAAI,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;QACnF,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC9C,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,OAAO;YAAE,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACjE,MAAM,OAAO,GAAwC,EAAE,CAAC;QACxD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YAChD,IAAI,MAAM,EAAE,MAAM;gBAAE,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,CAAC,SAAS,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QACjF,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;QACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5C,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;gBAC1D,IAAI,KAAK,IAAI,GAAG,CAAC,cAAc,EAAE,CAAC;oBAChC,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAC5D,IAAI,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;wBAAE,SAAS;oBAChC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBAClB,MAAM,CAAC,IAAI,CAAC;wBACV,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,gBAAgB;wBACrE,OAAO,EAAE,kBAAkB,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,cAAc,SAAS,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ;wBACjG,QAAQ,EAAE,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE;qBAC3E,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,oBAAoB;IACpB,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;IAEvH,MAAM,MAAM,GAAqC,EAAE,CAAC;IACpD,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;YAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;QACzC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;YAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAE,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAChF,CAAC;IAED,MAAM,MAAM,GAAG;QACb,OAAO,EAAE,OAAO,CAAC,MAAM;QACvB,OAAO,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;QACrD,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,MAAM;QAC3D,QAAQ,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,MAAM;QAC/D,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,MAAM;KAC1D,CAAC;IAEF,0EAA0E;IAC1E,0EAA0E;IAC1E,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,QAAQ,GAAG,CAAC,GAAG,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC;IAChF,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1E,6DAA6D;IAC7D,yDAAyD;IACzD,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,UAAU,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IAElF,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;AACjD,CAAC;AAED,SAAS,WAAW,CAAC,OAAe;IAClC,OAAO;QACL,MAAM,EAAE,EAAE;QACV,WAAW,EAAE,GAAG;QAChB,MAAM,EAAE,EAAE;QACV,MAAM,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;KAClE,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,KAAkB,EAAE,QAAkB;IACvD,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;QAC9B,IAAI,CAAC,IAAI,OAAO,CAAC,CAAC,cAAc,KAAK,QAAQ,EAAE,CAAC;YAC9C,GAAG,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CAAC,IAA4B,EAAE,UAA6B,EAAE;IACnG,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC;IACzB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,2BAA2B,IAAI,CAAC,WAAW,MAAM,CAAC,CAAC;IAC9D,KAAK,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,OAAO,cAAc,IAAI,CAAC,MAAM,CAAC,OAAO,YAAY,IAAI,CAAC,MAAM,CAAC,MAAM,cAAc,IAAI,CAAC,MAAM,CAAC,QAAQ,WAAW,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC;IAC3K,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED,iCAAiC;IACjC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAsB,CAAC;IAC5D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QACpC,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAC/B,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,KAAK,GAAG,CAAC,MAAM,YAAY,CAAC,CAAC;QAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;QAC5D,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;QAC1D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YACxF,KAAK,CAAC,IAAI,CAAC,OAAO,GAAG,IAAI,KAAK,CAAC,SAAS,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAChE,CAAC;QACD,IAAI,IAAI,IAAI,QAAQ,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,CAAC,cAAc,QAAQ,CAAC,MAAM,GAAG,IAAI,kBAAkB,CAAC,CAAC;QACrE,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,iDAAiD;IACjD,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpB,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC;QACpF,KAAK,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;IACtE,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC;QAC1F,KAAK,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;IACrD,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;IACxD,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;IAC7D,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;QAC5B,KAAK,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;IAC/D,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}

package/dist/src/analysis/saturation.d.ts

@@ -0,0 +1,85 @@
+/**
+ * Saturation curve analysis — answers "have I run enough samples?"
+ *
+ * Why this exists
+ * ---------------
+ * The most common omk-user question after "is the difference significant?"
+ * is "is N=30 enough, or should I keep running?". Without a principled
+ * answer, users either over-pay (running 200 when 50 was enough) or
+ * under-pay (calling skill effects null at N=30 when N=80 would have
+ * shown clearly significant — Bootstrap CI from Phase 1 just hasn't
+ * converged yet).
+ *
+ * Saturation analysis fits the right tool to the question: track a
+ * convergence-of-evidence metric across cumulative sample counts; when
+ * the metric stops moving meaningfully, you've saturated. Three metric
+ * choices are exposed because each has a failure mode:
+ *
+ *  - **slope** (simplest): mean's rate of change between consecutive
+ *    checkpoints. Easy to explain. Fragile to outlier samples.
+ *  - **bootstrap-ci-width** (default): CI shrinks as O(1/√N); when its
+ *    decay rate flattens, more samples buy little. Statistically
+ *    grounded; pairs naturally with Phase 1.
+ *  - **plateau-height**: range of mean across the last K checkpoints.
+ *    Conservative — slow to declare saturation, hard to fool.
+ *
+ * The function never says "saturated" off a single observation: a 3-
+ * window run-of-success is required. This guards against random dips
+ * that look like convergence but aren't.
+ */
+export type SaturationMethod = 'slope' | 'bootstrap-ci-width' | 'plateau-height';
+/**
+ * One observation in a saturation curve. `n` is the cumulative sample
+ * count at this checkpoint; `mean` and (optional) `ciWidth` come from
+ * applying the chosen metric to all samples up to and including `n`.
+ */
+export interface SaturationCheckpoint {
+    n: number;
+    mean: number;
+    ciWidth?: number;
+}
+export interface SaturationResult {
+    /** True only after threshold satisfied for `windowSize` consecutive transitions. */
+    saturated: boolean;
+    /** Sample count at which saturation was first declared. null if not saturated. */
+    atN: number | null;
+    /** 'high' for ≥ 50 cumulative samples; 'medium' for [20, 50); 'low' below 20. */
+    confidence: 'high' | 'medium' | 'low';
+    /** Method that produced the verdict. */
+    method: SaturationMethod;
+    /** Threshold used. */
+    threshold: number;
+    /** Per-checkpoint metric trace, useful for plotting. */
+    trace: Array<{
+        n: number;
+        metric: number;
+    }>;
+    /** Human-readable explanation of why we said yes / no. */
+    reason: string;
+}
+/**
+ * Compute saturation from a sequence of cumulative score arrays.
+ *
+ * `cumulativeScores[i]` is the array of all scores observed by checkpoint i
+ * (the i-th element is the LATEST cumulative slice; arrays grow). Each slice
+ * is fed to bootstrapMeanCI to get a (mean, CI) pair. The metric trajectory
+ * is then evaluated by the chosen method.
+ *
+ * @param cumulativeScores Cumulative score arrays in chronological order.
+ * @param method           Detection algorithm (default 'bootstrap-ci-width').
+ * @param threshold        Method-specific cutoff (defaults match the method).
+ * @param windowSize       Number of consecutive transitions that must all
+ *                          satisfy the threshold (default 3).
+ * @param bootstrapSamples Used only by 'bootstrap-ci-width' (default 1000).
+ * @param seed             Optional seed for reproducible CIs.
+ */
+export declare function findSaturationPoint(cumulativeScores: number[][], method?: SaturationMethod, threshold?: number, windowSize?: number, bootstrapSamples?: number, seed?: number): SaturationResult;
+/**
+ * Build cumulative score arrays from a per-run flat list. `runs[i]` is the
+ * array of per-sample scores from the i-th repeat. Returns an array where
+ * the j-th entry contains all scores from runs 0..j (concatenated).
+ *
+ * Use this when feeding `runMultiple` results to `findSaturationPoint`.
+ */
+export declare function buildCumulativeScores(runs: number[][]): number[][];
+//# sourceMappingURL=saturation.d.ts.map

package/dist/src/analysis/saturation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"saturation.d.ts","sourceRoot":"","sources":["../../../src/analysis/saturation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAIH,MAAM,MAAM,gBAAgB,GAAG,OAAO,GAAG,oBAAoB,GAAG,gBAAgB,CAAC;AAEjF;;;;GAIG;AACH,MAAM,WAAW,oBAAoB;IACnC,CAAC,EAAE,MAAM,CAAC;IACV,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,gBAAgB;IAC/B,oFAAoF;IACpF,SAAS,EAAE,OAAO,CAAC;IACnB,kFAAkF;IAClF,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,iFAAiF;IACjF,UAAU,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAC;IACtC,wCAAwC;IACxC,MAAM,EAAE,gBAAgB,CAAC;IACzB,sBAAsB;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,wDAAwD;IACxD,KAAK,EAAE,KAAK,CAAC;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC5C,0DAA0D;IAC1D,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,mBAAmB,CACjC,gBAAgB,EAAE,MAAM,EAAE,EAAE,EAC5B,MAAM,GAAE,gBAAuC,EAC/C,SAAS,CAAC,EAAE,MAAM,EAClB,UAAU,SAAI,EACd,gBAAgB,SAAO,EACvB,IAAI,CAAC,EAAE,MAAM,GACZ,gBAAgB,CA4GlB;AAUD;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,EAAE,CAQlE"}

package/dist/src/analysis/saturation.js

@@ -0,0 +1,174 @@
+/**
+ * Saturation curve analysis — answers "have I run enough samples?"
+ *
+ * Why this exists
+ * ---------------
+ * The most common omk-user question after "is the difference significant?"
+ * is "is N=30 enough, or should I keep running?". Without a principled
+ * answer, users either over-pay (running 200 when 50 was enough) or
+ * under-pay (calling skill effects null at N=30 when N=80 would have
+ * shown clearly significant — Bootstrap CI from Phase 1 just hasn't
+ * converged yet).
+ *
+ * Saturation analysis fits the right tool to the question: track a
+ * convergence-of-evidence metric across cumulative sample counts; when
+ * the metric stops moving meaningfully, you've saturated. Three metric
+ * choices are exposed because each has a failure mode:
+ *
+ *  - **slope** (simplest): mean's rate of change between consecutive
+ *    checkpoints. Easy to explain. Fragile to outlier samples.
+ *  - **bootstrap-ci-width** (default): CI shrinks as O(1/√N); when its
+ *    decay rate flattens, more samples buy little. Statistically
+ *    grounded; pairs naturally with Phase 1.
+ *  - **plateau-height**: range of mean across the last K checkpoints.
+ *    Conservative — slow to declare saturation, hard to fool.
+ *
+ * The function never says "saturated" off a single observation: a 3-
+ * window run-of-success is required. This guards against random dips
+ * that look like convergence but aren't.
+ */
+import { bootstrapMeanCI } from '../eval-core/bootstrap.js';
+/**
+ * Compute saturation from a sequence of cumulative score arrays.
+ *
+ * `cumulativeScores[i]` is the array of all scores observed by checkpoint i
+ * (the i-th element is the LATEST cumulative slice; arrays grow). Each slice
+ * is fed to bootstrapMeanCI to get a (mean, CI) pair. The metric trajectory
+ * is then evaluated by the chosen method.
+ *
+ * @param cumulativeScores Cumulative score arrays in chronological order.
+ * @param method           Detection algorithm (default 'bootstrap-ci-width').
+ * @param threshold        Method-specific cutoff (defaults match the method).
+ * @param windowSize       Number of consecutive transitions that must all
+ *                          satisfy the threshold (default 3).
+ * @param bootstrapSamples Used only by 'bootstrap-ci-width' (default 1000).
+ * @param seed             Optional seed for reproducible CIs.
+ */
+export function findSaturationPoint(cumulativeScores, method = 'bootstrap-ci-width', threshold, windowSize = 3, bootstrapSamples = 1000, seed) {
+    const trace = [];
+    const checkpoints = [];
+    for (const scores of cumulativeScores) {
+        if (scores.length === 0)
+            continue;
+        const ci = bootstrapMeanCI(scores, 0.05, bootstrapSamples, seed);
+        checkpoints.push({
+            n: scores.length,
+            mean: ci.estimate,
+            ciWidth: ci.high - ci.low,
+        });
+    }
+    // Default thresholds chosen for typical Likert-1-5 evaluations:
+    //  - slope: |Δmean| / |ΔN| < 0.005 (5% of one rubric step over 100 samples)
+    //  - bootstrap-ci-width: relative CI width shrink < 5% per checkpoint
+    //  - plateau-height: max-min(mean) < 0.1 over the last 3 windows
+    const t = threshold ?? defaultThreshold(method);
+    const lastN = checkpoints.length > 0 ? checkpoints[checkpoints.length - 1].n : 0;
+    const confidence = lastN >= 50 ? 'high'
+        : lastN >= 20 ? 'medium'
+            : 'low';
+    const baseShell = (overrides) => ({
+        saturated: false,
+        atN: null,
+        confidence,
+        method,
+        threshold: t,
+        trace,
+        reason: '',
+        ...overrides,
+    });
+    if (checkpoints.length < windowSize + 1) {
+        return baseShell({
+            reason: `数据点不足:有 ${checkpoints.length} 个,至少需要 ${windowSize + 1} 个`
+                + ` (windowSize=${windowSize})`,
+        });
+    }
+    // Compute per-method metric series. metric[i] is the "amount of change at
+    // transition i" — how much the metric moved from checkpoint i-1 to i.
+    // Rounded to 4 decimals to match bootstrap's precision and dodge IEEE-754
+    // surprises (e.g. 4.0 - 3.9 = 0.10000000000000009 > 0.1).
+    const round4 = (x) => Number.isFinite(x) ? Math.round(x * 10000) / 10000 : x;
+    let metricSeries;
+    let metricLabel;
+    switch (method) {
+        case 'slope': {
+            metricSeries = checkpoints.slice(1).map((cp, i) => ({
+                n: cp.n,
+                metric: round4(Math.abs(cp.mean - checkpoints[i].mean) / Math.max(1, cp.n - checkpoints[i].n)),
+            }));
+            metricLabel = 'mean 变化斜率 |Δmean/ΔN|';
+            break;
+        }
+        case 'bootstrap-ci-width': {
+            metricSeries = checkpoints.slice(1).map((cp, i) => {
+                const prev = checkpoints[i].ciWidth ?? 0;
+                const curr = cp.ciWidth ?? 0;
+                // Relative shrink: (prev - curr) / prev. Negative = CI grew (still finding signal).
+                const rel = prev > 0 ? Math.abs(prev - curr) / prev : 0;
+                return { n: cp.n, metric: round4(rel) };
+            });
+            metricLabel = 'CI 宽度相对衰减率';
+            break;
+        }
+        case 'plateau-height': {
+            // For each checkpoint i with i >= windowSize, compute max-min over the
+            // last windowSize+1 means. Earlier checkpoints get a Number.POSITIVE
+            // sentinel so they can never satisfy the threshold.
+            metricSeries = checkpoints.map((cp, i) => {
+                if (i < windowSize)
+                    return { n: cp.n, metric: Number.POSITIVE_INFINITY };
+                const window = checkpoints.slice(i - windowSize, i + 1).map((c) => c.mean);
+                const range = Math.max(...window) - Math.min(...window);
+                return { n: cp.n, metric: round4(range) };
+            });
+            metricLabel = 'mean 极差 (最近 K 窗口)';
+            break;
+        }
+    }
+    trace.push(...metricSeries);
+    // Saturated when the last `windowSize` consecutive metric values are all
+    // <= threshold. Walk forward; record the earliest qualifying checkpoint.
+    for (let i = windowSize - 1; i < metricSeries.length; i++) {
+        let allBelow = true;
+        for (let k = 0; k < windowSize; k++) {
+            if (metricSeries[i - k].metric > t) {
+                allBelow = false;
+                break;
+            }
+        }
+        if (allBelow) {
+            const atN = metricSeries[i].n;
+            return baseShell({
+                saturated: true,
+                atN,
+                reason: `连续 ${windowSize} 个窗口 ${metricLabel} ≤ ${t},于 N=${atN} 判定饱和`,
+            });
+        }
+    }
+    return baseShell({
+        reason: `${metricLabel} 尚未满足"连续 ${windowSize} 个窗口 ≤ ${t}"`,
+    });
+}
+function defaultThreshold(method) {
+    switch (method) {
+        case 'slope': return 0.005;
+        case 'bootstrap-ci-width': return 0.05;
+        case 'plateau-height': return 0.1;
+    }
+}
+/**
+ * Build cumulative score arrays from a per-run flat list. `runs[i]` is the
+ * array of per-sample scores from the i-th repeat. Returns an array where
+ * the j-th entry contains all scores from runs 0..j (concatenated).
+ *
+ * Use this when feeding `runMultiple` results to `findSaturationPoint`.
+ */
+export function buildCumulativeScores(runs) {
+    const acc = [];
+    const out = [];
+    for (const r of runs) {
+        acc.push(...r);
+        out.push([...acc]);
+    }
+    return out;
+}
+//# sourceMappingURL=saturation.js.map

package/dist/src/analysis/saturation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"saturation.js","sourceRoot":"","sources":["../../../src/analysis/saturation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAgC5D;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,mBAAmB,CACjC,gBAA4B,EAC5B,SAA2B,oBAAoB,EAC/C,SAAkB,EAClB,UAAU,GAAG,CAAC,EACd,gBAAgB,GAAG,IAAI,EACvB,IAAa;IAEb,MAAM,KAAK,GAA8B,EAAE,CAAC;IAC5C,MAAM,WAAW,GAA2B,EAAE,CAAC;IAE/C,KAAK,MAAM,MAAM,IAAI,gBAAgB,EAAE,CAAC;QACtC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAClC,MAAM,EAAE,GAAG,eAAe,CAAC,MAAM,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,CAAC,CAAC;QACjE,WAAW,CAAC,IAAI,CAAC;YACf,CAAC,EAAE,MAAM,CAAC,MAAM;YAChB,IAAI,EAAE,EAAE,CAAC,QAAQ;YACjB,OAAO,EAAE,EAAE,CAAC,IAAI,GAAG,EAAE,CAAC,GAAG;SAC1B,CAAC,CAAC;IACL,CAAC;IAED,gEAAgE;IAChE,4EAA4E;IAC5E,sEAAsE;IACtE,iEAAiE;IACjE,MAAM,CAAC,GAAG,SAAS,IAAI,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAChD,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACjF,MAAM,UAAU,GACd,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM;QACpB,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,QAAQ;YACxB,CAAC,CAAC,KAAK,CAAC;IAEV,MAAM,SAAS,GAAG,CAAC,SAAoC,EAAoB,EAAE,CAAC,CAAC;QAC7E,SAAS,EAAE,KAAK;QAChB,GAAG,EAAE,IAAI;QACT,UAAU;QACV,MAAM;QACN,SAAS,EAAE,CAAC;QACZ,KAAK;QACL,MAAM,EAAE,EAAE;QACV,GAAG,SAAS;KACb,CAAC,CAAC;IAEH,IAAI,WAAW,CAAC,MAAM,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC;QACxC,OAAO,SAAS,CAAC;YACf,MAAM,EAAE,WAAW,WAAW,CAAC,MAAM,WAAW,UAAU,GAAG,CAAC,IAAI;kBAC9D,gBAAgB,UAAU,GAAG;SAClC,CAAC,CAAC;IACL,CAAC;IAED,0EAA0E;IAC1E,sEAAsE;IACtE,0EAA0E;IAC1E,0DAA0D;IAC1D,MAAM,MAAM,GAAG,CAAC,CAAS,EAAU,EAAE,CACnC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACzD,IAAI,YAAkD,CAAC;IACvD,IAAI,WAAmB,CAAC;IACxB,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,OAAO,CAAC,CAAC,CAAC;YACb,YAAY,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBAClD,CAAC,EAAE,EAAE,CAAC,CAAC;gBACP,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;aAC/F,CAAC,CAAC,CAAC;YACJ,WAAW,GAAG,sBAAsB,CAAC;YACrC,MAAM;QACR,CAAC;QACD,KAAK,oBAAoB,CAAC,CAAC,CAAC;YAC1B,YAAY,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE;gBAChD,MAAM,IAAI,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC;gBACzC,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,IAAI,CAAC,CAAC;gBAC7B,oFAAoF;gBACpF,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;gBACxD,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1C,CAAC,CAAC,CAAC;YACH,WAAW,GAAG,YAAY,CAAC;YAC3B,MAAM;QACR,CAAC;QACD,KAAK,gBAAgB,CAAC,CAAC,CAAC;YACtB,uEAAuE;YACvE,qEAAqE;YACrE,oDAAoD;YACpD,YAAY,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE;gBACvC,IAAI,CAAC,GAAG,UAAU;oBAAE,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,iBAAiB,EAAE,CAAC;gBACzE,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,GAAG,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBAC3E,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;gBACxD,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5C,CAAC,CAAC,CAAC;YACH,WAAW,GAAG,mBAAmB,CAAC;YAClC,MAAM;QACR,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;IAE5B,yEAAyE;IACzE,yEAAyE;IACzE,KAAK,IAAI,CAAC,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1D,IAAI,QAAQ,GAAG,IAAI,CAAC;QACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,IAAI,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAAC,QAAQ,GAAG,KAAK,CAAC;gBAAC,MAAM;YAAC,CAAC;QAClE,CAAC;QACD,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9B,OAAO,SAAS,CAAC;gBACf,SAAS,EAAE,IAAI;gBACf,GAAG;gBACH,MAAM,EAAE,MAAM,UAAU,QAAQ,WAAW,MAAM,CAAC,QAAQ,GAAG,OAAO;aACrE,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,SAAS,CAAC;QACf,MAAM,EAAE,GAAG,WAAW,YAAY,UAAU,UAAU,CAAC,GAAG;KAC3D,CAAC,CAAC;AACL,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAwB;IAChD,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,OAAO,CAAC,CAAC,OAAO,KAAK,CAAC;QAC3B,KAAK,oBAAoB,CAAC,CAAC,OAAO,IAAI,CAAC;QACvC,KAAK,gBAAgB,CAAC,CAAC,OAAO,GAAG,CAAC;IACpC,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAgB;IACpD,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,GAAG,GAAe,EAAE,CAAC;IAC3B,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACf,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;IACrB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}