cognitive-core 0.2.5 → 0.3.1

package/dist/eval/playbook-usage-report.js

@@ -0,0 +1,222 @@
+import { existsSync, readdirSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+export function renderPlaybookUsageReport(traceRoot, opts = {}) {
+    const learningArm = opts.learningArm ?? 'full';
+    const baselineArm = opts.baselineArm ?? 'no-playbooks';
+    const learningDir = join(traceRoot, learningArm);
+    const baselineDir = join(traceRoot, baselineArm);
+    if (!existsSync(learningDir) || !existsSync(baselineDir)) {
+        throw new Error(`playbook usage report requires trace subdirectories '${learningArm}' and '${baselineArm}' under ${traceRoot}`);
+    }
+    const learning = readTaskDiagnostics(learningDir);
+    const baseline = readTaskDiagnostics(baselineDir);
+    const baselineByIndex = new Map(baseline.map((d) => [d.taskIndex, d]));
+    const firstInjected = learning.find((d) => d.injection.playbookCount > 0)?.taskIndex;
+    const postStart = opts.postExtractionStartIndex ?? firstInjected ?? 0;
+    let injectedTasks = 0;
+    let injectedWins = 0;
+    let injectedLosses = 0;
+    let discordantWins = 0;
+    let discordantLosses = 0;
+    let paired = 0;
+    let postLearningWins = 0;
+    let postBaselineWins = 0;
+    let postPairs = 0;
+    let previews = 0;
+    const byPlaybook = new Map();
+    for (const task of learning) {
+        const base = baselineByIndex.get(task.taskIndex);
+        if (base)
+            paired++;
+        if (task.outcome.outputPreview)
+            previews++;
+        const wasInjected = task.injection.playbookCount > 0;
+        if (wasInjected) {
+            injectedTasks++;
+            if (task.outcome.success)
+                injectedWins++;
+            else
+                injectedLosses++;
+        }
+        const learningWon = task.outcome.success;
+        const baselineWon = base?.outcome.success;
+        if (base && wasInjected && learningWon && !baselineWon)
+            discordantWins++;
+        if (base && wasInjected && !learningWon && baselineWon)
+            discordantLosses++;
+        if (base && task.taskIndex >= postStart) {
+            postPairs++;
+            if (learningWon)
+                postLearningWins++;
+            if (baselineWon)
+                postBaselineWins++;
+        }
+        for (const pb of task.injection.playbooks ?? []) {
+            const agg = byPlaybook.get(pb.id) ?? {
+                id: pb.id,
+                name: pb.name,
+                surfaced: 0,
+                wins: 0,
+                losses: 0,
+                discordantWins: 0,
+                discordantLosses: 0,
+                scoreSum: 0,
+                methods: new Set(),
+                previewEvidenceTasks: 0,
+                previewKeywordHits: 0,
+            };
+            agg.surfaced++;
+            if (task.outcome.success)
+                agg.wins++;
+            else
+                agg.losses++;
+            if (base && task.outcome.success && !base.outcome.success)
+                agg.discordantWins++;
+            if (base && !task.outcome.success && base.outcome.success)
+                agg.discordantLosses++;
+            agg.scoreSum += pb.score;
+            if (pb.extractionMethod)
+                agg.methods.add(pb.extractionMethod);
+            const evidence = previewEvidence(task.injection.blob, pb.name, task.outcome.outputPreview);
+            if (evidence.keywordHits >= 3)
+                agg.previewEvidenceTasks++;
+            agg.previewKeywordHits += evidence.keywordHits;
+            byPlaybook.set(pb.id, agg);
+        }
+    }
+    const pct = (n, d) => (d === 0 ? 'n/a' : `${((n / d) * 100).toFixed(1)}%`);
+    const lines = [
+        '# Playbook Usage Report',
+        '',
+        `**Trace root:** \`${traceRoot}\``,
+        `**Comparison:** \`${learningArm}\` vs \`${baselineArm}\``,
+        '',
+        '> This is a surfacing/outcome audit. It proves which playbooks reached the solver and how paired outcomes changed; it is not by itself proof that the model followed every tactic.',
+        '',
+        '## Summary',
+        '',
+        `- Learning tasks: ${learning.length}`,
+        `- Paired baseline tasks: ${paired}`,
+        `- Tasks with playbooks injected: ${injectedTasks}/${learning.length} (${pct(injectedTasks, learning.length)})`,
+        `- Injected-task success: ${injectedWins}/${injectedTasks} (${pct(injectedWins, injectedTasks)})`,
+        `- Discordant wins with playbooks injected: ${discordantWins}`,
+        `- Discordant losses with playbooks injected: ${discordantLosses}`,
+        `- Post-extraction paired success from index ${postStart}: ${postLearningWins}/${postPairs} learning vs ${postBaselineWins}/${postPairs} baseline`,
+        `- Diagnostics with output previews for deeper audit: ${previews}/${learning.length}`,
+        '',
+        '## Per-Playbook Outcome',
+        '',
+        '| Playbook | Method | Surfaced | Win rate | Preview evidence | Discordant wins | Discordant losses | Avg score |',
+        '|----------|--------|---------:|---------:|-----------------:|----------------:|------------------:|----------:|',
+    ];
+    const aggregates = [...byPlaybook.values()].sort((a, b) => b.discordantWins - a.discordantWins ||
+        b.surfaced - a.surfaced ||
+        a.name.localeCompare(b.name));
+    if (aggregates.length === 0) {
+        lines.push('| _(none)_ |  | 0 | n/a | n/a | 0 | 0 | n/a |');
+    }
+    else {
+        for (const agg of aggregates) {
+            lines.push(`| ${escapeCell(agg.name)} | ${escapeCell([...agg.methods].sort().join(', ') || 'unknown')} | ` +
+                `${agg.surfaced} | ${pct(agg.wins, agg.surfaced)} | ` +
+                `${agg.previewEvidenceTasks}/${agg.surfaced} (${agg.previewKeywordHits} hits) | ${agg.discordantWins} | ` +
+                `${agg.discordantLosses} | ${(agg.scoreSum / agg.surfaced).toFixed(3)} |`);
+        }
+    }
+    lines.push('', '## Task Pairs', '', '| # | Task | Learning | Baseline | Injected | Playbooks |', '|--:|------|----------|----------|---------:|-----------|');
+    for (const task of learning) {
+        const base = baselineByIndex.get(task.taskIndex);
+        const taskName = task.taskId.split('/').pop() ?? task.taskId;
+        const names = (task.injection.playbooks ?? []).map((p) => p.name).join(', ');
+        lines.push(`| ${task.taskIndex} | ${escapeCell(taskName)} | ${task.outcome.success ? 'PASS' : 'fail'} | ` +
+            `${base ? (base.outcome.success ? 'PASS' : 'fail') : 'n/a'} | ${task.injection.playbookCount} | ` +
+            `${escapeCell(names || '-')} |`);
+    }
+    return lines.join('\n');
+}
+function readTaskDiagnostics(traceDir) {
+    const out = [];
+    for (const file of readdirSync(traceDir)
+        .filter((f) => f.endsWith('.jsonl'))
+        .sort()) {
+        if (file === 'learning.jsonl')
+            continue;
+        const raw = readFileSync(join(traceDir, file), 'utf-8');
+        for (const line of raw.split('\n')) {
+            if (!line.trim())
+                continue;
+            const entry = JSON.parse(line);
+            if (entry.type === 'diagnostics') {
+                out.push(entry);
+            }
+        }
+    }
+    return out.sort((a, b) => a.taskIndex - b.taskIndex);
+}
+function escapeCell(value) {
+    return value.replace(/\|/g, '\\|');
+}
+function previewEvidence(injectedBlob, playbookName, outputPreview) {
+    if (!outputPreview)
+        return { keywordHits: 0 };
+    const section = findPlaybookSection(injectedBlob, playbookName);
+    if (!section)
+        return { keywordHits: 0 };
+    const guidanceTokens = new Set(tokenize(section));
+    const previewTokens = new Set(tokenize(outputPreview));
+    let hits = 0;
+    for (const token of guidanceTokens) {
+        if (previewTokens.has(token))
+            hits++;
+    }
+    return { keywordHits: hits };
+}
+function findPlaybookSection(blob, playbookName) {
+    const sections = blob.split(/\n### /);
+    for (const section of sections) {
+        const normalized = section.startsWith('### ') ? section.slice(4) : section;
+        if (normalized.startsWith(`${playbookName} `) || normalized.startsWith(`${playbookName}\n`)) {
+            return normalized;
+        }
+    }
+    return undefined;
+}
+const STOPWORDS = new Set([
+    'about',
+    'after',
+    'again',
+    'against',
+    'also',
+    'apply',
+    'before',
+    'below',
+    'candidate',
+    'check',
+    'current',
+    'from',
+    'have',
+    'into',
+    'only',
+    'should',
+    'task',
+    'that',
+    'their',
+    'these',
+    'this',
+    'when',
+    'where',
+    'with',
+    'without',
+    'write',
+    'using',
+    'must',
+    'will',
+    'your',
+]);
+function tokenize(value) {
+    return value
+        .toLowerCase()
+        .split(/[^a-z0-9_]+/)
+        .filter((token) => token.length >= 5 && !STOPWORDS.has(token));
+}
+//# sourceMappingURL=playbook-usage-report.js.map

package/dist/eval/playbook-usage-report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"playbook-usage-report.js","sourceRoot":"","sources":["../../src/eval/playbook-usage-report.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAChE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAuBjC,MAAM,UAAU,yBAAyB,CACvC,SAAiB,EACjB,OAAmC,EAAE;IAErC,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,MAAM,CAAC;IAC/C,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,cAAc,CAAC;IACvD,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IACjD,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IAEjD,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QACzD,MAAM,IAAI,KAAK,CACb,wDAAwD,WAAW,UAAU,WAAW,WAAW,SAAS,EAAE,CAC/G,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAC;IAClD,MAAM,QAAQ,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAC;IAClD,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACvE,MAAM,aAAa,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,aAAa,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC;IACrF,MAAM,SAAS,GAAG,IAAI,CAAC,wBAAwB,IAAI,aAAa,IAAI,CAAC,CAAC;IAEtE,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,MAAM,UAAU,GAAG,IAAI,GAAG,EAA6B,CAAC;IAExD,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACjD,IAAI,IAAI;YAAE,MAAM,EAAE,CAAC;QACnB,IAAI,IAAI,CAAC,OAAO,CAAC,aAAa;YAAE,QAAQ,EAAE,CAAC;QAE3C,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,aAAa,GAAG,CAAC,CAAC;QACrD,IAAI,WAAW,EAAE,CAAC;YAChB,aAAa,EAAE,CAAC;YAChB,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO;gBAAE,YAAY,EAAE,CAAC;;gBACpC,cAAc,EAAE,CAAC;QACxB,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;QACzC,MAAM,WAAW,GAAG,IAAI,EAAE,OAAO,CAAC,OAAO,CAAC;QAC1C,IAAI,IAAI,IAAI,WAAW,IAAI,WAAW,IAAI,CAAC,WAAW;YAAE,cAAc,EAAE,CAAC;QACzE,IAAI,IAAI,IAAI,WAAW,IAAI,CAAC,WAAW,IAAI,WAAW;YAAE,gBAAgB,EAAE,CAAC;QAE3E,IAAI,IAAI,IAAI,IAAI,CAAC,SAAS,IAAI,SAAS,EAAE,CAAC;YACxC,SAAS,EAAE,CAAC;YACZ,IAAI,WAAW;gBAAE,gBAAgB,EAAE,CAAC;YACpC,IAAI,WAAW;gBAAE,gBAAgB,EAAE,CAAC;QACtC,CAAC;QAED,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,SAAS,IAAI,EAAE,EAAE,CAAC;YAChD,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI;gBACnC,EAAE,EAAE,EAAE,CAAC,EAAE;gBACT,IAAI,EAAE,EAAE,CAAC,IAAI;gBACb,QAAQ,EAAE,CAAC;gBACX,IAAI,EAAE,CAAC;gBACP,MAAM,EAAE,CAAC;gBACT,cAAc,EAAE,CAAC;gBACjB,gBAAgB,EAAE,CAAC;gBACnB,QAAQ,EAAE,CAAC;gBACX,OAAO,EAAE,IAAI,GAAG,EAAU;gBAC1B,oBAAoB,EAAE,CAAC;gBACvB,kBAAkB,EAAE,CAAC;aACtB,CAAC;YACF,GAAG,CAAC,QAAQ,EAAE,CAAC;YACf,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO;gBAAE,GAAG,CAAC,IAAI,EAAE,CAAC;;gBAChC,GAAG,CAAC,MAAM,EAAE,CAAC;YAClB,IAAI,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO;gBAAE,GAAG,CAAC,cAAc,EAAE,CAAC;YAChF,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO;gBAAE,GAAG,CAAC,gBAAgB,EAAE,CAAC;YAClF,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC,KAAK,CAAC;YACzB,IAAI,EAAE,CAAC,gBAAgB;gBAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,gBAAgB,CAAC,CAAC;YAC9D,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;YAC3F,IAAI,QAAQ,CAAC,WAAW,IAAI,CAAC;gBAAE,GAAG,CAAC,oBAAoB,EAAE,CAAC;YAC1D,GAAG,CAAC,kBAAkB,IAAI,QAAQ,CAAC,WAAW,CAAC;YAC/C,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC3F,MAAM,KAAK,GAAa;QACtB,yBAAyB;QACzB,EAAE;QACF,qBAAqB,SAAS,IAAI;QAClC,qBAAqB,WAAW,WAAW,WAAW,IAAI;QAC1D,EAAE;QACF,oLAAoL;QACpL,EAAE;QACF,YAAY;QACZ,EAAE;QACF,qBAAqB,QAAQ,CAAC,MAAM,EAAE;QACtC,4BAA4B,MAAM,EAAE;QACpC,oCAAoC,aAAa,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC,aAAa,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG;QAC/G,4BAA4B,YAAY,IAAI,aAAa,KAAK,GAAG,CAAC,YAAY,EAAE,aAAa,CAAC,GAAG;QACjG,8CAA8C,cAAc,EAAE;QAC9D,gDAAgD,gBAAgB,EAAE;QAClE,+CAA+C,SAAS,KAAK,gBAAgB,IAAI,SAAS,gBAAgB,gBAAgB,IAAI,SAAS,WAAW;QAClJ,wDAAwD,QAAQ,IAAI,QAAQ,CAAC,MAAM,EAAE;QACrF,EAAE;QACF,yBAAyB;QACzB,EAAE;QACF,kHAAkH;QAClH,kHAAkH;KACnH,CAAC;IAEF,MAAM,UAAU,GAAG,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAC9C,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,CAAC,CAAC,cAAc,GAAG,CAAC,CAAC,cAAc;QACnC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ;QACvB,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAC/B,CAAC;IACF,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,KAAK,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;IAC9D,CAAC;SAAM,CAAC;QACN,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;YAC7B,KAAK,CAAC,IAAI,CACR,KAAK,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,UAAU,CAAC,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,KAAK;gBAC7F,GAAG,GAAG,CAAC,QAAQ,MAAM,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,QAAQ,CAAC,KAAK;gBACrD,GAAG,GAAG,CAAC,oBAAoB,IAAI,GAAG,CAAC,QAAQ,KAAK,GAAG,CAAC,kBAAkB,YAAY,GAAG,CAAC,cAAc,KAAK;gBACzG,GAAG,GAAG,CAAC,gBAAgB,MAAM,CAAC,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAC5E,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CACR,EAAE,EACF,eAAe,EACf,EAAE,EACF,2DAA2D,EAC3D,2DAA2D,CAC5D,CAAC;IACF,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACjD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC;QAC7D,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7E,KAAK,CAAC,IAAI,CACR,KAAK,IAAI,CAAC,SAAS,MAAM,UAAU,CAAC,QAAQ,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,KAAK;YAC5F,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,MAAM,IAAI,CAAC,SAAS,CAAC,aAAa,KAAK;YACjG,GAAG,UAAU,CAAC,KAAK,IAAI,GAAG,CAAC,IAAI,CAClC,CAAC;IACJ,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,mBAAmB,CAAC,QAAgB;IAC3C,MAAM,GAAG,GAAsB,EAAE,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,QAAQ,CAAC;SACrC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;SACnC,IAAI,EAAE,EAAE,CAAC;QACV,IAAI,IAAI,KAAK,gBAAgB;YAAE,SAAS;QACxC,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;QACxD,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACnC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;gBAAE,SAAS;YAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAA4B,CAAC;YAC1D,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;gBACjC,GAAG,CAAC,IAAI,CAAC,KAAmC,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC;AACvD,CAAC;AAED,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;AACrC,CAAC;AAED,SAAS,eAAe,CACtB,YAAoB,EACpB,YAAoB,EACpB,aAAsB;IAEtB,IAAI,CAAC,aAAa;QAAE,OAAO,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC;IAC9C,MAAM,OAAO,GAAG,mBAAmB,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;IAChE,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC;IACxC,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IAClD,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC;IACvD,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;QACnC,IAAI,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC;YAAE,IAAI,EAAE,CAAC;IACvC,CAAC;IACD,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;AAC/B,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAY,EAAE,YAAoB;IAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACtC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QAC3E,IAAI,UAAU,CAAC,UAAU,CAAC,GAAG,YAAY,GAAG,CAAC,IAAI,UAAU,CAAC,UAAU,CAAC,GAAG,YAAY,IAAI,CAAC,EAAE,CAAC;YAC5F,OAAO,UAAU,CAAC;QACpB,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC;IACxB,OAAO;IACP,OAAO;IACP,OAAO;IACP,SAAS;IACT,MAAM;IACN,OAAO;IACP,QAAQ;IACR,OAAO;IACP,WAAW;IACX,OAAO;IACP,SAAS;IACT,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,QAAQ;IACR,MAAM;IACN,MAAM;IACN,OAAO;IACP,OAAO;IACP,MAAM;IACN,MAAM;IACN,OAAO;IACP,MAAM;IACN,SAAS;IACT,OAAO;IACP,OAAO;IACP,MAAM;IACN,MAAM;IACN,MAAM;CACP,CAAC,CAAC;AAEH,SAAS,QAAQ,CAAC,KAAa;IAC7B,OAAO,KAAK;SACT,WAAW,EAAE;SACb,KAAK,CAAC,aAAa,CAAC;SACpB,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;AACnE,CAAC"}

package/dist/eval/report.d.ts

@@ -0,0 +1,28 @@
+import type { EvalResult } from './types.js';
+import type { AblationResult } from './ablation.js';
+import type { PlaybookExtractionDiagnostics } from '../learning/unified-pipeline.js';
+import { type TaskDiagnostics } from './diagnostics.js';
+export declare function renderMarkdown(result: EvalResult): string;
+export declare function renderAblationMarkdown(result: AblationResult): string;
+export declare function renderAblationJson(result: AblationResult): string;
+/**
+ * Benchmark-specific detail sections appended to the main report.
+ * Dispatches on `benchmarkName` to render per-family breakdowns,
+ * transfer matrices, convergence info, or gain metrics as appropriate.
+ */
+export declare function renderBenchmarkDetails(result: EvalResult): string;
+export declare function renderJson(result: EvalResult): string;
+/**
+ * Load all TaskDiagnostics entries from a trace directory.
+ * Each .jsonl file may contain multiple lines; lines with type='diagnostics'
+ * are parsed as TaskDiagnostics. Returns a flat array sorted by taskIndex.
+ */
+export declare function loadDiagnosticsFromTraceDir(traceDir: string): TaskDiagnostics[];
+export declare function loadLearningDiagnosticsFromTraceDir(traceDir: string): PlaybookExtractionDiagnostics[];
+/**
+ * Render a human-readable Markdown diagnostics report from a trace directory.
+ * Splits diagnostics by arm (learning / no-learning), computes funnels,
+ * identifies discordant pairs, and formats everything as Markdown.
+ */
+export declare function renderDiagnosticsReport(traceDir: string): string;
+//# sourceMappingURL=report.d.ts.map

package/dist/eval/report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/eval/report.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAA+B,UAAU,EAAc,MAAM,YAAY,CAAC;AACtF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACpD,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAC;AAErF,OAAO,EAIL,KAAK,eAAe,EACrB,MAAM,kBAAkB,CAAC;AAoF1B,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAuEzD;AAaD,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,cAAc,GAAG,MAAM,CAqDrE;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,cAAc,GAAG,MAAM,CAsBjE;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAkBjE;AAuKD,wBAAgB,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAiCrD;AAMD;;;;GAIG;AACH,wBAAgB,2BAA2B,CAAC,QAAQ,EAAE,MAAM,GAAG,eAAe,EAAE,CAyB/E;AAED,wBAAgB,mCAAmC,CACjD,QAAQ,EAAE,MAAM,GACf,6BAA6B,EAAE,CAwBjC;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAmGhE"}