oh-my-knowledge 0.18.0 → 0.20.0

package/dist/src/analysis/failure-clusterer.d.ts

@@ -0,0 +1,96 @@
+/**
+ * Failure clustering — turn a list of "what failed" into "why it failed".
+ *
+ * Why this exists
+ * ---------------
+ * Senior-engineer pain point: 50 samples ran, 14 failed. Reading 14 failure
+ * cases one by one is tedious; the user wants the tool to say "8 of these
+ * are tool-call errors, 4 are refusals, 2 are formatting errors", with a
+ * suggested fix per cluster.
+ *
+ * This module:
+ *
+ *  1. Selects failed samples from a report (compositeScore < threshold OR
+ *     ok=false on at least one variant).
+ *  2. Builds a compact failure description per sample (failed assertion
+ *     types + judge reason snippet + tool failure pattern).
+ *  3. Sends the full set to a single LLM call asking it to (a) propose
+ *     N cluster labels, (b) assign each failure to a cluster, (c) suggest
+ *     a fix per cluster.
+ *  4. Returns the structured cluster report. Pure function — no I/O,
+ *     accepts the executor as an argument.
+ *
+ * Design choice: single-call LLM clustering rather than embedding-based
+ * k-means. Reasons:
+ *
+ *  - Failure descriptions are short (~50-200 tokens each), so 14 of them
+ *    fit comfortably in one prompt. A 50-failure batch is also fine.
+ *  - The LLM can produce HUMAN-READABLE cluster labels ("tool call error",
+ *    "premature refusal") rather than numeric centroids — exactly what
+ *    the user needs to act on.
+ *  - One call = one judge cost = predictable. Embedding + clustering
+ *    needs N+1 calls minimum.
+ *  - Acceptable inaccuracy: clusters are advisory, not load-bearing. We
+ *    document this clearly and surface raw failures alongside.
+ *
+ * The clustering prompt is structured to be JSON-only output for parsing
+ * stability. We do NOT include length-debias instructions — clustering is
+ * a categorization task, not a quality scoring task, so the length-debias
+ * directive is inapplicable.
+ */
+import type { ExecutorFn, Report } from '../types.js';
+export interface FailureClusterRequest {
+    report: Report;
+    /** Executor for the clustering LLM call. */
+    executor: ExecutorFn;
+    /** Model id to use for clustering. */
+    judgeModel: string;
+    /** Max number of clusters. Default 5; the LLM may produce fewer. */
+    maxClusters?: number;
+    /** compositeScore < this OR ok=false counts as failure. Default 3. */
+    failureThreshold?: number;
+    /** Cap on how many failures to feed the LLM (selects worst by score). Default 50. */
+    maxFailuresFed?: number;
+}
+export interface FailureCase {
+    sample_id: string;
+    variant: string;
+    /** Composite score (0 if errored). */
+    score: number;
+    /** Whether the executor itself errored (ok=false). */
+    errored: boolean;
+    /** Description summarizing what went wrong, fed to the LLM. */
+    description: string;
+}
+export interface FailureCluster {
+    /** Human-readable cluster label, in the report's primary language. */
+    label: string;
+    /** Free-text root cause analysis from the LLM. */
+    rootCause: string;
+    /** Suggested fix the user can apply. */
+    suggestedFix: string;
+    /** Sample IDs (with variant) belonging to this cluster. */
+    members: Array<{
+        sample_id: string;
+        variant: string;
+    }>;
+}
+export interface FailureClusterReport {
+    /** All failures considered (after threshold + cap). */
+    failures: FailureCase[];
+    /** Clusters returned by the LLM, sorted by member count desc. */
+    clusters: FailureCluster[];
+    /** Failures the LLM didn't put in any cluster (label = "other"). */
+    unclassified: Array<{
+        sample_id: string;
+        variant: string;
+    }>;
+    /** USD cost of the clustering call. */
+    clusterCostUSD: number;
+    /** Truncation flag — set when more failures existed than maxFailuresFed. */
+    truncated: boolean;
+    totalFailures: number;
+}
+export declare function clusterFailures(req: FailureClusterRequest): Promise<FailureClusterReport>;
+export declare function formatFailureClusterReport(r: FailureClusterReport): string;
+//# sourceMappingURL=failure-clusterer.d.ts.map

package/dist/src/analysis/failure-clusterer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"failure-clusterer.d.ts","sourceRoot":"","sources":["../../../src/analysis/failure-clusterer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,EAA8B,MAAM,aAAa,CAAC;AAElF,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,4CAA4C;IAC5C,QAAQ,EAAE,UAAU,CAAC;IACrB,sCAAsC;IACtC,UAAU,EAAE,MAAM,CAAC;IACnB,oEAAoE;IACpE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,sEAAsE;IACtE,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,qFAAqF;IACrF,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,WAAW;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,sDAAsD;IACtD,OAAO,EAAE,OAAO,CAAC;IACjB,+DAA+D;IAC/D,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,sEAAsE;IACtE,KAAK,EAAE,MAAM,CAAC;IACd,kDAAkD;IAClD,SAAS,EAAE,MAAM,CAAC;IAClB,wCAAwC;IACxC,YAAY,EAAE,MAAM,CAAC;IACrB,2DAA2D;IAC3D,OAAO,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACxD;AAED,MAAM,WAAW,oBAAoB;IACnC,uDAAuD;IACvD,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,iEAAiE;IACjE,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,oEAAoE;IACpE,YAAY,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC5D,uCAAuC;IACvC,cAAc,EAAE,MAAM,CAAC;IACvB,4EAA4E;IAC5E,SAAS,EAAE,OAAO,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;CACvB;AAeD,wBAAsB,eAAe,CAAC,GAAG,EAAE,qBAAqB,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAkF/F;AAwID,wBAAgB,0BAA0B,CAAC,CAAC,EAAE,oBAAoB,GAAG,MAAM,CAsC1E"}

package/dist/src/analysis/failure-clusterer.js

@@ -0,0 +1,298 @@
+/**
+ * Failure clustering — turn a list of "what failed" into "why it failed".
+ *
+ * Why this exists
+ * ---------------
+ * Senior-engineer pain point: 50 samples ran, 14 failed. Reading 14 failure
+ * cases one by one is tedious; the user wants the tool to say "8 of these
+ * are tool-call errors, 4 are refusals, 2 are formatting errors", with a
+ * suggested fix per cluster.
+ *
+ * This module:
+ *
+ *  1. Selects failed samples from a report (compositeScore < threshold OR
+ *     ok=false on at least one variant).
+ *  2. Builds a compact failure description per sample (failed assertion
+ *     types + judge reason snippet + tool failure pattern).
+ *  3. Sends the full set to a single LLM call asking it to (a) propose
+ *     N cluster labels, (b) assign each failure to a cluster, (c) suggest
+ *     a fix per cluster.
+ *  4. Returns the structured cluster report. Pure function — no I/O,
+ *     accepts the executor as an argument.
+ *
+ * Design choice: single-call LLM clustering rather than embedding-based
+ * k-means. Reasons:
+ *
+ *  - Failure descriptions are short (~50-200 tokens each), so 14 of them
+ *    fit comfortably in one prompt. A 50-failure batch is also fine.
+ *  - The LLM can produce HUMAN-READABLE cluster labels ("tool call error",
+ *    "premature refusal") rather than numeric centroids — exactly what
+ *    the user needs to act on.
+ *  - One call = one judge cost = predictable. Embedding + clustering
+ *    needs N+1 calls minimum.
+ *  - Acceptable inaccuracy: clusters are advisory, not load-bearing. We
+ *    document this clearly and surface raw failures alongside.
+ *
+ * The clustering prompt is structured to be JSON-only output for parsing
+ * stability. We do NOT include length-debias instructions — clustering is
+ * a categorization task, not a quality scoring task, so the length-debias
+ * directive is inapplicable.
+ */
+const DEFAULT_MAX_CLUSTERS = 5;
+const DEFAULT_FAILURE_THRESHOLD = 3;
+const DEFAULT_MAX_FED = 50;
+export async function clusterFailures(req) {
+    const { report, executor, judgeModel, maxClusters = DEFAULT_MAX_CLUSTERS, failureThreshold = DEFAULT_FAILURE_THRESHOLD, maxFailuresFed = DEFAULT_MAX_FED, } = req;
+    const allFailures = collectFailures(report, failureThreshold);
+    // Sort by severity (errored > low score) then take up to maxFailuresFed.
+    allFailures.sort((a, b) => {
+        if (a.errored !== b.errored)
+            return a.errored ? -1 : 1;
+        return a.score - b.score;
+    });
+    const truncated = allFailures.length > maxFailuresFed;
+    const fed = allFailures.slice(0, maxFailuresFed);
+    if (fed.length === 0) {
+        return {
+            failures: [], clusters: [], unclassified: [],
+            clusterCostUSD: 0, truncated: false, totalFailures: 0,
+        };
+    }
+    // Need at least 2 failures for clustering to be meaningful — with 1 we
+    // simply emit a single trivial cluster without calling the LLM.
+    if (fed.length === 1) {
+        return {
+            failures: fed,
+            clusters: [{
+                    label: '单条失败',
+                    rootCause: '只有 1 条失败,无法聚类',
+                    suggestedFix: '直接看下面的失败描述',
+                    members: [{ sample_id: fed[0].sample_id, variant: fed[0].variant }],
+                }],
+            unclassified: [],
+            clusterCostUSD: 0,
+            truncated: false,
+            totalFailures: 1,
+        };
+    }
+    const prompt = buildClusterPrompt(fed, maxClusters);
+    const result = await executor({
+        model: judgeModel,
+        system: '你是 LLM 评测失败案例分类员。逐条分类,只返回 JSON。',
+        prompt,
+    });
+    if (!result.ok) {
+        return {
+            failures: fed,
+            clusters: [],
+            unclassified: fed.map((f) => ({ sample_id: f.sample_id, variant: f.variant })),
+            clusterCostUSD: result.costUSD || 0,
+            truncated,
+            totalFailures: allFailures.length,
+        };
+    }
+    const parsed = parseClusterResponse(result.output ?? '');
+    const clusters = normalizeClusters(parsed, fed);
+    // Anything fed but not assigned to a cluster lands in unclassified.
+    const claimed = new Set();
+    for (const c of clusters) {
+        for (const m of c.members)
+            claimed.add(`${m.sample_id}::${m.variant}`);
+    }
+    const unclassified = fed
+        .filter((f) => !claimed.has(`${f.sample_id}::${f.variant}`))
+        .map((f) => ({ sample_id: f.sample_id, variant: f.variant }));
+    clusters.sort((a, b) => b.members.length - a.members.length);
+    return {
+        failures: fed,
+        clusters,
+        unclassified,
+        clusterCostUSD: result.costUSD || 0,
+        truncated,
+        totalFailures: allFailures.length,
+    };
+}
+function collectFailures(report, threshold) {
+    const variants = report.meta?.variants ?? [];
+    const out = [];
+    for (const entry of report.results ?? []) {
+        for (const v of variants) {
+            const r = entry.variants?.[v];
+            if (!r)
+                continue;
+            const score = typeof r.compositeScore === 'number' ? r.compositeScore : 0;
+            const errored = r.ok === false;
+            if (!errored && score >= threshold)
+                continue;
+            out.push({
+                sample_id: entry.sample_id,
+                variant: v,
+                score,
+                errored,
+                description: describeFailure(entry, v, r),
+            });
+        }
+    }
+    return out;
+}
+/**
+ * Build a compact textual description of one failure for the clustering LLM.
+ * Keeps it under ~250 tokens so 50 failures fit in a single ~12k token prompt.
+ */
+function describeFailure(entry, variant, r) {
+    const parts = [];
+    if (r.ok === false) {
+        parts.push(`EXEC_ERROR: ${truncate(r.error ?? 'unknown error', 120)}`);
+    }
+    if (typeof r.compositeScore === 'number' && r.compositeScore < 3) {
+        parts.push(`compositeScore=${r.compositeScore.toFixed(2)}`);
+    }
+    // Failed assertions — top 5 to keep size bounded.
+    const failed = (r.assertions?.details ?? [])
+        .filter((d) => !d.passed)
+        .slice(0, 5)
+        .map((d) => `${d.type}(${truncate(String(d.value), 40)})`);
+    if (failed.length > 0) {
+        parts.push(`failed_assertions=[${failed.join(', ')}]`);
+    }
+    // Judge reasoning snippet.
+    if (r.llmReason)
+        parts.push(`judge_reason="${truncate(r.llmReason, 200)}"`);
+    // Tool failures (agent traces).
+    const toolFailures = (r.toolCalls ?? []).filter((tc) => !tc.success).slice(0, 3);
+    if (toolFailures.length > 0) {
+        parts.push(`failed_tools=[${toolFailures.map((tc) => `${tc.tool}: ${truncate(String(tc.output ?? ''), 80)}`).join('; ')}]`);
+    }
+    // Output preview if nothing else informative.
+    if (parts.length === 0 && r.outputPreview) {
+        parts.push(`output_preview="${truncate(r.outputPreview, 200)}"`);
+    }
+    return `[${entry.sample_id}@${variant}] ${parts.join(' | ')}`;
+}
+function truncate(s, n) {
+    if (s.length <= n)
+        return s;
+    return s.slice(0, n - 1) + '…';
+}
+function buildClusterPrompt(failures, maxClusters) {
+    const lines = [];
+    lines.push(`分析下面 ${failures.length} 条 LLM 评测失败案例,把它们分到 ≤ ${maxClusters} 个 cluster。`);
+    lines.push('');
+    lines.push('## 失败案例 (每行一条):');
+    for (const f of failures) {
+        lines.push(f.description);
+    }
+    lines.push('');
+    lines.push('## 任务');
+    lines.push(`1. 根据失败模式提出最多 ${maxClusters} 个 cluster (例如:工具调用错误 / 拒答 / 格式错 / 事实错 / 其它)`);
+    lines.push('2. 把每条失败分到一个 cluster (用 sample_id@variant 引用)');
+    lines.push('3. 每个 cluster 给出根因 + 一条具体修复建议');
+    lines.push('');
+    lines.push('## 返回格式 (严格 JSON,不要 markdown 代码块):');
+    lines.push('{');
+    lines.push('  "clusters": [');
+    lines.push('    {');
+    lines.push('      "label": "<简短中文标签>",');
+    lines.push('      "rootCause": "<根因说明,1-2 句>",');
+    lines.push('      "suggestedFix": "<修复建议,1-2 句,要可立即操作>",');
+    lines.push('      "members": [{"sample_id": "...", "variant": "..."}, ...]');
+    lines.push('    }');
+    lines.push('  ]');
+    lines.push('}');
+    return lines.join('\n');
+}
+function parseClusterResponse(text) {
+    const trimmed = text.trim();
+    // Tolerate accidental ```json fences from the model.
+    const cleaned = trimmed.replace(/^```(?:json)?\s*/i, '').replace(/\s*```$/i, '');
+    const match = cleaned.match(/\{[\s\S]*\}/);
+    if (!match)
+        return { clusters: [] };
+    try {
+        return JSON.parse(match[0]);
+    }
+    catch {
+        return { clusters: [] };
+    }
+}
+function normalizeClusters(parsed, fed) {
+    const validIds = new Set(fed.map((f) => `${f.sample_id}::${f.variant}`));
+    const clusters = parsed.clusters ?? [];
+    const out = [];
+    for (const c of clusters) {
+        const label = typeof c.label === 'string' && c.label ? c.label : '未命名';
+        const rootCause = typeof c.rootCause === 'string' ? c.rootCause : '';
+        const suggestedFix = typeof c.suggestedFix === 'string' ? c.suggestedFix : '';
+        const memberArr = Array.isArray(c.members) ? c.members : [];
+        const members = [];
+        for (const m of memberArr) {
+            let sample_id;
+            let variant;
+            if (typeof m === 'string') {
+                // Tolerate "sample_id@variant" string form.
+                const parts = m.split('@');
+                if (parts.length === 2) {
+                    sample_id = parts[0];
+                    variant = parts[1];
+                }
+            }
+            else if (m && typeof m === 'object') {
+                sample_id = typeof m.sample_id === 'string' ? m.sample_id : undefined;
+                variant = typeof m.variant === 'string' ? m.variant : undefined;
+            }
+            if (!sample_id || !variant)
+                continue;
+            if (!validIds.has(`${sample_id}::${variant}`))
+                continue; // hallucinated member
+            members.push({ sample_id, variant });
+        }
+        if (members.length > 0) {
+            out.push({ label, rootCause, suggestedFix, members });
+        }
+    }
+    return out;
+}
+export function formatFailureClusterReport(r) {
+    const lines = [];
+    lines.push('');
+    lines.push(`  失败聚类 — ${r.totalFailures} 条失败${r.truncated ? ` (取最差 ${r.failures.length} 条做聚类)` : ''}`);
+    lines.push('');
+    if (r.clusters.length === 0) {
+        lines.push('  (聚类失败,以下是原始失败列表)');
+        for (const f of r.failures.slice(0, 10)) {
+            lines.push(`    ${f.sample_id}@${f.variant}: ${truncate(f.description, 120)}`);
+        }
+        if (r.failures.length > 10)
+            lines.push(`    ... 还有 ${r.failures.length - 10} 条`);
+        lines.push('');
+        return lines.join('\n');
+    }
+    for (const c of r.clusters) {
+        lines.push(`  [${c.label}] ${c.members.length} 条`);
+        if (c.rootCause)
+            lines.push(`    根因: ${c.rootCause}`);
+        if (c.suggestedFix)
+            lines.push(`    建议: ${c.suggestedFix}`);
+        const display = c.members.slice(0, 5);
+        for (const m of display) {
+            lines.push(`      - ${m.sample_id}@${m.variant}`);
+        }
+        if (c.members.length > display.length) {
+            lines.push(`      ... 还有 ${c.members.length - display.length} 条`);
+        }
+        lines.push('');
+    }
+    if (r.unclassified.length > 0) {
+        lines.push(`  [未分类] ${r.unclassified.length} 条`);
+        for (const m of r.unclassified.slice(0, 5)) {
+            lines.push(`      - ${m.sample_id}@${m.variant}`);
+        }
+        if (r.unclassified.length > 5)
+            lines.push(`      ... 还有 ${r.unclassified.length - 5} 条`);
+        lines.push('');
+    }
+    lines.push(`  聚类调用 cost: $${r.clusterCostUSD.toFixed(6)}`);
+    lines.push('');
+    return lines.join('\n');
+}
+//# sourceMappingURL=failure-clusterer.js.map

package/dist/src/analysis/failure-clusterer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"failure-clusterer.js","sourceRoot":"","sources":["../../../src/analysis/failure-clusterer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AA+DH,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAC/B,MAAM,yBAAyB,GAAG,CAAC,CAAC;AACpC,MAAM,eAAe,GAAG,EAAE,CAAC;AAE3B,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,GAA0B;IAC9D,MAAM,EACJ,MAAM,EAAE,QAAQ,EAAE,UAAU,EAC5B,WAAW,GAAG,oBAAoB,EAClC,gBAAgB,GAAG,yBAAyB,EAC5C,cAAc,GAAG,eAAe,GACjC,GAAG,GAAG,CAAC;IAER,MAAM,WAAW,GAAG,eAAe,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC9D,yEAAyE;IACzE,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACxB,IAAI,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO;YAAE,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,OAAO,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;IAC3B,CAAC,CAAC,CAAC;IACH,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,GAAG,cAAc,CAAC;IACtD,MAAM,GAAG,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;IAEjD,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrB,OAAO;YACL,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,EAAE;YAC5C,cAAc,EAAE,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC;SACtD,CAAC;IACJ,CAAC;IAED,uEAAuE;IACvE,gEAAgE;IAChE,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrB,OAAO;YACL,QAAQ,EAAE,GAAG;YACb,QAAQ,EAAE,CAAC;oBACT,KAAK,EAAE,MAAM;oBACb,SAAS,EAAE,eAAe;oBAC1B,YAAY,EAAE,YAAY;oBAC1B,OAAO,EAAE,CAAC,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;iBACpE,CAAC;YACF,YAAY,EAAE,EAAE;YAChB,cAAc,EAAE,CAAC;YACjB,SAAS,EAAE,KAAK;YAChB,aAAa,EAAE,CAAC;SACjB,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,kBAAkB,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;IACpD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC;QAC5B,KAAK,EAAE,UAAU;QACjB,MAAM,EAAE,iCAAiC;QACzC,MAAM;KACP,CAAC,CAAC;IAEH,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,OAAO;YACL,QAAQ,EAAE,GAAG;YACb,QAAQ,EAAE,EAAE;YACZ,YAAY,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YAC9E,cAAc,EAAE,MAAM,CAAC,OAAO,IAAI,CAAC;YACnC,SAAS;YACT,aAAa,EAAE,WAAW,CAAC,MAAM;SAClC,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,oBAAoB,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAG,iBAAiB,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAEhD,oEAAoE;IACpE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO;YAAE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;IACzE,CAAC;IACD,MAAM,YAAY,GAAG,GAAG;SACrB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;SAC3D,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAEhE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAE7D,OAAO;QACL,QAAQ,EAAE,GAAG;QACb,QAAQ;QACR,YAAY;QACZ,cAAc,EAAE,MAAM,CAAC,OAAO,IAAI,CAAC;QACnC,SAAS;QACT,aAAa,EAAE,WAAW,CAAC,MAAM;KAClC,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,MAAc,EAAE,SAAiB;IACxD,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,EAAE,QAAQ,IAAI,EAAE,CAAC;IAC7C,MAAM,GAAG,GAAkB,EAAE,CAAC;IAC9B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC;QACzC,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,MAAM,CAAC,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;YAC9B,IAAI,CAAC,CAAC;gBAAE,SAAS;YACjB,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,cAAc,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1E,MAAM,OAAO,GAAG,CAAC,CAAC,EAAE,KAAK,KAAK,CAAC;YAC/B,IAAI,CAAC,OAAO,IAAI,KAAK,IAAI,SAAS;gBAAE,SAAS;YAC7C,GAAG,CAAC,IAAI,CAAC;gBACP,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,OAAO,EAAE,CAAC;gBACV,KAAK;gBACL,OAAO;gBACP,WAAW,EAAE,eAAe,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;aAC1C,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;GAGG;AACH,SAAS,eAAe,CAAC,KAAkB,EAAE,OAAe,EAAE,CAAgB;IAC5E,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,CAAC,CAAC,EAAE,KAAK,KAAK,EAAE,CAAC;QACnB,KAAK,CAAC,IAAI,CAAC,eAAe,QAAQ,CAAC,CAAC,CAAC,KAAK,IAAI,eAAe,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IACzE,CAAC;IACD,IAAI,OAAO,CAAC,CAAC,cAAc,KAAK,QAAQ,IAAI,CAAC,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC;QACjE,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC9D,CAAC;IACD,kDAAkD;IAClD,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,UAAU,EAAE,OAAO,IAAI,EAAE,CAAC;SACzC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;SACxB,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;IAC7D,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,KAAK,CAAC,IAAI,CAAC,sBAAsB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzD,CAAC;IACD,2BAA2B;IAC3B,IAAI,CAAC,CAAC,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,iBAAiB,QAAQ,CAAC,CAAC,CAAC,SAAS,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5E,gCAAgC;IAChC,MAAM,YAAY,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACjF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,KAAK,CAAC,IAAI,CAAC,iBAAiB,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,EAAE,CAAC,IAAI,KAAK,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC9H,CAAC;IACD,8CAA8C;IAC9C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC,aAAa,EAAE,CAAC;QAC1C,KAAK,CAAC,IAAI,CAAC,mBAAmB,QAAQ,CAAC,CAAC,CAAC,aAAa,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;IACnE,CAAC;IACD,OAAO,IAAI,KAAK,CAAC,SAAS,IAAI,OAAO,KAAK,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;AAChE,CAAC;AAED,SAAS,QAAQ,CAAC,CAAS,EAAE,CAAS;IACpC,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAC5B,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC;AACjC,CAAC;AAED,SAAS,kBAAkB,CAAC,QAAuB,EAAE,WAAmB;IACtE,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,QAAQ,QAAQ,CAAC,MAAM,yBAAyB,WAAW,aAAa,CAAC,CAAC;IACrF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC9B,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;IAC5B,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpB,KAAK,CAAC,IAAI,CAAC,iBAAiB,WAAW,8CAA8C,CAAC,CAAC;IACvF,KAAK,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;IAC5D,KAAK,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;IAC5C,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;IACjD,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAChB,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC9B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpB,KAAK,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;IACzC,KAAK,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;IACjD,KAAK,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;IAC3D,KAAK,CAAC,IAAI,CAAC,gEAAgE,CAAC,CAAC;IAC7E,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpB,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClB,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAChB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,oBAAoB,CAAC,IAAY;IACxC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,qDAAqD;IACrD,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IACjF,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAC3C,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACpC,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAyB,CAAC;IACtD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IAC1B,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CAAC,MAA4B,EAAE,GAAkB;IACzE,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IACzE,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC;IACvC,MAAM,GAAG,GAAqB,EAAE,CAAC;IACjC,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;QACvE,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;QACrE,MAAM,YAAY,GAAG,OAAO,CAAC,CAAC,YAAY,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9E,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5D,MAAM,OAAO,GAA8B,EAAE,CAAC;QAC9C,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;YAC1B,IAAI,SAA6B,CAAC;YAClC,IAAI,OAA2B,CAAC;YAChC,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;gBAC1B,4CAA4C;gBAC5C,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAC3B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAAC,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;oBAAC,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;gBAAC,CAAC;YACvE,CAAC;iBAAM,IAAI,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;gBACtC,SAAS,GAAG,OAAO,CAAC,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;gBACtE,OAAO,GAAG,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;YAClE,CAAC;YACD,IAAI,CAAC,SAAS,IAAI,CAAC,OAAO;gBAAE,SAAS;YACrC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,SAAS,KAAK,OAAO,EAAE,CAAC;gBAAE,SAAS,CAAC,sBAAsB;YAC/E,OAAO,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC;QACvC,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,CAAuB;IAChE,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,aAAa,OAAO,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,MAAM,QAAQ,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACtG,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,KAAK,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QACjC,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YACxC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACjF,CAAC;QACD,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,EAAE;YAAE,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,EAAE,IAAI,CAAC,CAAC;QACjF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;QACnD,IAAI,CAAC,CAAC,SAAS;YAAE,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;QACtD,IAAI,CAAC,CAAC,YAAY;YAAE,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC;QAC5D,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACtC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QACpD,CAAC;QACD,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;YACtC,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;QACpE,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IACD,IAAI,CAAC,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,YAAY,CAAC,MAAM,IAAI,CAAC,CAAC;QACjD,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;YAC3C,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QACpD,CAAC;QACD,IAAI,CAAC,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC;QACzF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC3D,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}

package/dist/src/analysis/sample-diagnostics.d.ts

@@ -0,0 +1,78 @@
+/**
+ * Per-sample quality diagnostics.
+ *
+ * Why this exists
+ * ---------------
+ * `report-diagnostics.ts` answers "what's wrong with the report overall" —
+ * aggregate-level insights about assertions and variant comparisons. This
+ * module answers a complementary question: "which specific samples in my
+ * eval-samples set are problematic and should be fixed before I trust any
+ * conclusion drawn on this report?"
+ *
+ * Senior-engineer pain point: 100 eval-samples typically have 20-40 that
+ * (a) all variants get the same score on (no discriminative power), or
+ * (b) are near-duplicates of another sample (data redundancy inflating N), or
+ * (c) have ambiguous rubric (judge gives wildly different scores on repeats), or
+ * (d) are cost / latency outliers (one sample spikes the budget by itself), or
+ * (e) fail across the board (likely broken sample, not a model deficiency).
+ *
+ * The CLI `omk bench diagnose <reportId>` surfaces these so users can iterate
+ * the SAMPLE SET as much as they iterate the SKILL.
+ *
+ * Output shape: a flat list of issues, each tagged with severity and the
+ * minimal evidence needed to act ("here are the matching prompts"). Issues are
+ * sorted by severity then by sample_id so the report is stable across runs.
+ */
+import type { Report, Sample } from '../types.js';
+export type SampleIssueKind = 'flat_scores' | 'all_pass' | 'all_fail' | 'near_duplicate' | 'ambiguous_rubric' | 'cost_outlier' | 'latency_outlier' | 'error_prone';
+export interface SampleIssue {
+    sample_id: string;
+    severity: 'error' | 'warning' | 'info';
+    kind: SampleIssueKind;
+    message: string;
+    /** Minimal evidence to make the issue actionable. */
+    evidence: Record<string, unknown>;
+}
+export interface SampleDiagnosticReport {
+    /** All issues sorted by (severity desc, sample_id asc). */
+    issues: SampleIssue[];
+    /** 0-100 quality score for the sample set as a whole. Lower = more issues. */
+    healthScore: number;
+    /** Sample IDs grouped by issue kind for quick consumption. */
+    byKind: Partial<Record<SampleIssueKind, string[]>>;
+    /** Quick-glance counters. */
+    totals: {
+        samples: number;
+        flagged: number;
+        errors: number;
+        warnings: number;
+        infos: number;
+    };
+}
+export interface DiagnoseOptions {
+    /** Minimum ROUGE-1 between two prompts to flag near-duplicate. Default 0.7. */
+    duplicateRouge?: number;
+    /** Judge stddev threshold (across judgeRepeat samples) to flag ambiguous. Default 1.0. */
+    ambiguousStddev?: number;
+    /** Cost outlier multiplier vs median. Default 3 (≥ 3× median is flagged). */
+    costOutlierK?: number;
+    /** Latency outlier multiplier vs median. Default 3. */
+    latencyOutlierK?: number;
+    /** Score-spread threshold below which a sample is "flat". Default 0.5. */
+    flatThreshold?: number;
+    /**
+     * Original Sample objects, used for near-duplicate detection. When omitted
+     * (Report alone) the duplicate check is skipped — Report doesn't carry
+     * prompts, only score/cost/latency. The CLI handler reads samples via
+     * report.meta.request.samplesPath and passes them in.
+     */
+    samples?: Sample[];
+}
+export declare function diagnoseSamples(report: Report, options?: DiagnoseOptions): SampleDiagnosticReport;
+/**
+ * Plain-text formatter for `omk bench diagnose <reportId>`.
+ */
+export declare function formatSampleDiagnostics(diag: SampleDiagnosticReport, options?: {
+    topN?: number;
+}): string;
+//# sourceMappingURL=sample-diagnostics.d.ts.map

package/dist/src/analysis/sample-diagnostics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sample-diagnostics.d.ts","sourceRoot":"","sources":["../../../src/analysis/sample-diagnostics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,KAAK,EAAE,MAAM,EAAe,MAAM,EAAE,MAAM,aAAa,CAAC;AAG/D,MAAM,MAAM,eAAe,GACvB,aAAa,GACb,UAAU,GACV,UAAU,GACV,gBAAgB,GAChB,kBAAkB,GAClB,cAAc,GACd,iBAAiB,GACjB,aAAa,CAAC;AAElB,MAAM,WAAW,WAAW;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,OAAO,GAAG,SAAS,GAAG,MAAM,CAAC;IACvC,IAAI,EAAE,eAAe,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,qDAAqD;IACrD,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAED,MAAM,WAAW,sBAAsB;IACrC,2DAA2D;IAC3D,MAAM,EAAE,WAAW,EAAE,CAAC;IACtB,8EAA8E;IAC9E,WAAW,EAAE,MAAM,CAAC;IACpB,8DAA8D;IAC9D,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,eAAe,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;IACnD,6BAA6B;IAC7B,MAAM,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;CAC/F;AAED,MAAM,WAAW,eAAe;IAC9B,+EAA+E;IAC/E,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,0FAA0F;IAC1F,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,6EAA6E;IAC7E,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,uDAAuD;IACvD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,0EAA0E;IAC1E,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAYD,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB,GAAG,sBAAsB,CAmKrG;AAsBD;;GAEG;AACH,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,sBAAsB,EAAE,OAAO,GAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,MAAM,CAoD7G"}