@wix/pathgrade 0.29.0

package/dist/reporters/cli.js

@@ -0,0 +1,131 @@
+import * as path from 'path';
+import { fmt, header } from '../utils/cli.js';
+import { loadReports } from './loader.js';
+// ─── Main ──────────────────────────────────────────────────
+export async function runCliPreview(resultsDir, opts) {
+    const resolved = path.resolve(resultsDir);
+    let entries = await loadReports(resolved);
+    if (opts?.filter) {
+        const pattern = opts.filter.toLowerCase();
+        entries = entries.filter(e => e.task?.toLowerCase().includes(pattern));
+    }
+    if (opts?.last && opts.last > 0) {
+        entries = entries.slice(0, opts.last);
+    }
+    if (!entries.length) {
+        console.log(`\n  ${fmt.dim('No reports found in')} ${resolved}\n`);
+        return;
+    }
+    console.log(`\n${fmt.bold('pathgrade preview')}  ${fmt.dim(`${entries.length} reports from ${resolved}`)}\n`);
+    for (const { file, ...report } of entries) {
+        const passRate = report.pass_rate ?? 0;
+        const isPass = passRate >= 0.5;
+        const trials = report.trials || [];
+        const avgDur = trials.reduce((s, t) => s + (t.duration_ms || 0), 0) / (trials.length || 1);
+        const totalTokens = trials.reduce((s, t) => s + (t.input_tokens || 0) + (t.output_tokens || 0) + (t.conversation_input_tokens || 0) + (t.conversation_output_tokens || 0), 0);
+        // ── Report header
+        const status = isPass ? fmt.pass('PASS') : fmt.fail('FAIL');
+        header(`${status}  ${report.task}`);
+        // Timestamp from filename
+        const ts = file.match(/\d{4}-\d{2}-\d{2}T[\d-]+/)?.[0]?.replace(/-(\d{2})-(\d{2})-/g, ':$1:$2:') || '';
+        if (ts)
+            console.log(`    ${fmt.dim(ts)}`);
+        console.log();
+        // ── Summary metrics
+        const metrics = [
+            ['Pass Rate', `${(passRate * 100).toFixed(1)}%`],
+            ['pass@k', report.pass_at_k != null ? `${(report.pass_at_k * 100).toFixed(1)}%` : '—'],
+            ['pass^k', report.pass_pow_k != null ? `${(report.pass_pow_k * 100).toFixed(1)}%` : '—'],
+            ['Avg Duration', `${(avgDur / 1000).toFixed(1)}s`],
+            ['Total Tokens', `~${totalTokens}`],
+            ['Skills', report.skills_used?.join(', ') || 'none'],
+        ];
+        for (const [label, value] of metrics) {
+            console.log(`    ${fmt.dim(label.padEnd(14))} ${fmt.bold(value)}`);
+        }
+        console.log();
+        // ── Trials
+        for (const trial of trials) {
+            const tp = trial.reward >= 0.5;
+            const trialStatus = tp ? fmt.pass('PASS') : fmt.fail('FAIL');
+            const reward = fmt.bold(trial.reward.toFixed(2));
+            const dur = `${((trial.duration_ms || 0) / 1000).toFixed(1)}s`;
+            const cmds = `${trial.n_commands || 0} cmds`;
+            const scorers = (trial.scorer_results || []).map((g) => {
+                const scoreStr = g.score.toFixed(1);
+                const colored = g.score >= 0.5 ? fmt.green(scoreStr) : fmt.red(scoreStr);
+                const statusLabel = formatScorerStatus(g.status);
+                return `${fmt.dim(g.scorer_type)}${statusLabel ? ` ${statusLabel}` : ''} ${colored}`;
+            }).join('  ');
+            const toolEvents = (trial.session_log || []).filter((entry) => entry.type === 'tool_event');
+            const toolSuffix = toolEvents.length ? `  ${fmt.dim(toolEvents.length + ' tool events')}` : '';
+            const judgeToolCalls = (trial.session_log || []).filter((entry) => entry.type === 'judge_tool_call');
+            const judgeToolNames = judgeToolCalls
+                .map((e) => e.judge_tool_call?.name)
+                .filter(Boolean);
+            const uniqueJudgeTools = Array.from(new Set(judgeToolNames));
+            const judgeToolSuffix = judgeToolCalls.length
+                ? `  ${fmt.dim(`${judgeToolCalls.length} judge tool calls (${uniqueJudgeTools.join(', ')})`)}`
+                : '';
+            const convSuffix = trial.conversation
+                ? `  ${fmt.dim(`${trial.conversation.total_turns} turns`)}  ${fmt.dim(trial.conversation.completion_reason)}`
+                : '';
+            const trialLabel = trial.name || `${trial.trial_id}`;
+            console.log(`    ${fmt.dim(trialLabel.padEnd(4))} ${trialStatus}  ${reward}  ${fmt.dim(dur.padEnd(7))} ${fmt.dim(cmds.padEnd(7))} ${scorers}${convSuffix}${toolSuffix}${judgeToolSuffix}`);
+        }
+        console.log();
+        const nonOkScorers = trials.flatMap((trial) => (trial.scorer_results || [])
+            .filter((g) => g.status && g.status !== 'ok')
+            .map((g) => ({ trial, scorer: g })));
+        if (nonOkScorers.length > 0) {
+            console.log(`    ${fmt.bold('Scorer Statuses')}`);
+            for (const entry of nonOkScorers) {
+                console.log(`    ${fmt.dim(`trial ${entry.trial.trial_id}`)} ${formatScorerStatus(entry.scorer.status)} ${fmt.dim(entry.scorer.scorer_type)} ${fmt.dim(entry.scorer.details.substring(0, 100))}`);
+            }
+            console.log();
+        }
+        // ── LLM scorer details
+        const hasLlm = trials.some((t) => t.scorer_results?.some((g) => g.scorer_type === 'llm_rubric'));
+        if (hasLlm) {
+            for (const trial of trials) {
+                const llmScorers = (trial.scorer_results || []).filter((g) => g.scorer_type === 'llm_rubric');
+                for (const g of llmScorers) {
+                    const scoreStr = g.score >= 0.5 ? fmt.green(g.score.toFixed(2)) : fmt.red(g.score.toFixed(2));
+                    console.log(`    ${fmt.dim(`trial ${trial.trial_id}`)} ${scoreStr} ${fmt.dim(g.details.substring(0, 100))}`);
+                }
+            }
+            console.log();
+        }
+        // ── Step scorer details (conversation trials)
+        const hasStepScorers = trials.some((t) => t.conversation?.turns?.some((turn) => (turn.step_scorer_results?.length ?? 0) > 0));
+        if (hasStepScorers) {
+            console.log(`    ${fmt.bold('Step Scorers')}`);
+            for (const trial of trials) {
+                if (!trial.conversation?.turns)
+                    continue;
+                for (const turn of trial.conversation.turns) {
+                    if (!turn.step_scorer_results?.length)
+                        continue;
+                    for (const g of turn.step_scorer_results) {
+                        const scoreStr = g.score >= 0.5 ? fmt.green(g.score.toFixed(2)) : fmt.red(g.score.toFixed(2));
+                        const statusLabel = formatScorerStatus(g.status);
+                        console.log(`    ${fmt.dim(`trial ${trial.trial_id} turn ${turn.turn_number}`)} ${scoreStr} ${fmt.dim(g.scorer_type)}${statusLabel ? ` ${statusLabel}` : ''}  ${fmt.dim(g.details.substring(0, 80))}`);
+                    }
+                }
+            }
+            console.log();
+        }
+        console.log(`    ${fmt.dim(file)}`);
+        console.log();
+    }
+}
+function formatScorerStatus(status) {
+    switch (status) {
+        case 'error':
+            return '[ERROR]';
+        case 'skipped':
+            return '[SKIPPED]';
+        default:
+            return '';
+    }
+}

package/dist/reporters/diagnostics.d.ts

@@ -0,0 +1,46 @@
+import type { LogEntry } from '../types.js';
+import type { ReactionFiredEntry, TurnDetail } from '../sdk/types.js';
+import { type RuntimePolicyAuditEntry } from '../sdk/runtime-policy.js';
+export interface DiagnosticsScorer {
+    name: string;
+    type: string;
+    score: number;
+    weight: number;
+    details?: string;
+    status?: 'ok' | 'error' | 'skipped';
+}
+export interface DiagnosticsTurnDetail extends TurnDetail {
+    apiRetries: number;
+}
+export type DiagnosticsRuntimePolicyEntry = RuntimePolicyAuditEntry;
+export interface DiagnosticsReport {
+    turns: number;
+    totalDurationMs: number;
+    completionReason?: string;
+    completionDetail?: string;
+    score?: number;
+    turnDetails: DiagnosticsTurnDetail[];
+    reactionsFired: ReactionFiredEntry[];
+    runtimePoliciesApplied: DiagnosticsRuntimePolicyEntry[];
+    recommendedTimeoutMs: number;
+    warnings: string[];
+    scorers: Array<DiagnosticsScorer & {
+        status: 'ok' | 'error' | 'skipped';
+    }>;
+}
+export interface BuildDiagnosticsReportInput {
+    completionReason?: string;
+    completionDetail?: string;
+    score?: number;
+    turnDetails?: TurnDetail[];
+    reactionsFired?: ReactionFiredEntry[];
+    scorers?: DiagnosticsScorer[];
+    log: LogEntry[];
+}
+export interface FormatDiagnosticsOptions {
+    verbose?: boolean;
+    currentTimeoutMs?: number;
+}
+export declare function buildDiagnosticsReport(input: BuildDiagnosticsReportInput): DiagnosticsReport;
+export declare function formatDiagnosticsSummary(report: DiagnosticsReport): string;
+export declare function formatDiagnostics(report: DiagnosticsReport, opts?: FormatDiagnosticsOptions): string;

package/dist/reporters/diagnostics.js

@@ -0,0 +1,147 @@
+import { extractRuntimePolicyAuditEntries } from '../sdk/runtime-policy.js';
+export function buildDiagnosticsReport(input) {
+    const baseTurnDetails = (input.turnDetails && input.turnDetails.length > 0)
+        ? [...input.turnDetails]
+        : deriveTurnDetailsFromLog(input.log);
+    const retriesByTurn = countRetriesByTurn(input.log);
+    const turnDetails = baseTurnDetails
+        .sort((a, b) => a.turn - b.turn)
+        .map((detail) => ({
+        ...detail,
+        apiRetries: retriesByTurn.get(detail.turn) ?? 0,
+    }));
+    const warnings = turnDetails
+        .filter((detail) => detail.outputLines > 500)
+        .map((detail) => `Turn ${detail.turn}: output exceeded 500 lines (${detail.outputLines} lines)`);
+    return {
+        turns: turnDetails.length,
+        totalDurationMs: turnDetails.reduce((sum, detail) => sum + detail.durationMs, 0),
+        completionReason: input.completionReason,
+        completionDetail: input.completionDetail,
+        score: input.score,
+        turnDetails,
+        reactionsFired: input.reactionsFired ?? [],
+        runtimePoliciesApplied: extractRuntimePolicyAuditEntries(input.log),
+        recommendedTimeoutMs: recommendTimeoutMs(turnDetails),
+        warnings,
+        scorers: (input.scorers ?? []).map((scorer) => ({
+            ...scorer,
+            status: scorer.status ?? 'ok',
+        })),
+    };
+}
+export function formatDiagnosticsSummary(report) {
+    const turnsLabel = report.turns === 1 ? '1 turn' : `${report.turns} turns`;
+    const durationLabel = `${(report.totalDurationMs / 1000).toFixed(1)}s`;
+    const completion = report.completionReason ?? 'unknown';
+    const scoreLabel = report.score == null ? 'n/a' : report.score.toFixed(2);
+    return `${turnsLabel}  ${durationLabel}  ${completion}  score ${scoreLabel}`;
+}
+export function formatDiagnostics(report, opts = {}) {
+    if (opts.verbose === false) {
+        return formatDiagnosticsSummary(report);
+    }
+    const lines = [
+        'Unified Diagnostics',
+        `Summary: ${formatDiagnosticsSummary(report)}`,
+    ];
+    if (report.completionDetail && report.completionReason === 'agent_crashed') {
+        lines.push(`Agent crashed: ${report.completionDetail}`);
+    }
+    else if (report.completionDetail
+        && (report.completionReason === 'error' || report.completionReason === 'timeout')) {
+        lines.push(`Error: ${report.completionDetail}`);
+    }
+    lines.push('', 'Turns:');
+    for (const detail of report.turnDetails) {
+        lines.push(`  Turn ${detail.turn}: ${(detail.durationMs / 1000).toFixed(1)}s, ${detail.outputLines} lines, ${detail.outputChars} chars, api retries: ${detail.apiRetries}`);
+    }
+    if (report.reactionsFired.length > 0) {
+        lines.push('', 'Reactions:');
+        for (const reaction of report.reactionsFired) {
+            lines.push(`  Turn ${reaction.turn}: reaction ${reaction.reactionIndex} ${reaction.pattern} -> ${reaction.reply}`);
+        }
+    }
+    if (report.runtimePoliciesApplied.length > 0) {
+        lines.push('', 'Runtime policies:');
+        for (const policy of report.runtimePoliciesApplied) {
+            lines.push(`  ${policy.id}@${policy.version} applied on turns ${policy.turns.join(', ')}`);
+        }
+    }
+    if (report.scorers.length > 0) {
+        lines.push('', 'Scorers:');
+        for (const scorer of report.scorers) {
+            const detailSuffix = scorer.details ? ` ${scorer.details}` : '';
+            lines.push(`  [${scorer.status}] ${scorer.name} (${scorer.type}) ${scorer.score.toFixed(2)}${detailSuffix}`);
+        }
+    }
+    if (report.warnings.length > 0) {
+        lines.push('', 'Warnings:');
+        for (const warning of report.warnings) {
+            lines.push(`  ${warning}`);
+        }
+    }
+    lines.push('', formatTimeoutRecommendation(report.recommendedTimeoutMs, opts.currentTimeoutMs));
+    return lines.join('\n');
+}
+function deriveTurnDetailsFromLog(log) {
+    return log
+        .filter((entry) => entry.type === 'agent_result'
+        && entry.synthetic_blocked_prompt !== true
+        && typeof entry.turn_number === 'number'
+        && typeof entry.duration_ms === 'number'
+        && typeof entry.output_lines === 'number'
+        && typeof entry.output_chars === 'number')
+        .map((entry) => ({
+        turn: entry.turn_number,
+        durationMs: entry.duration_ms,
+        outputLines: entry.output_lines,
+        outputChars: entry.output_chars,
+    }));
+}
+function countRetriesByTurn(log) {
+    const retriesByTurn = new Map();
+    let pendingRetries = 0;
+    for (const entry of log) {
+        if (isRetryEntry(entry)) {
+            pendingRetries++;
+            continue;
+        }
+        if (entry.type === 'agent_result' && typeof entry.turn_number === 'number') {
+            if (pendingRetries > 0) {
+                retriesByTurn.set(entry.turn_number, (retriesByTurn.get(entry.turn_number) ?? 0) + pendingRetries);
+                pendingRetries = 0;
+            }
+            if (!retriesByTurn.has(entry.turn_number)) {
+                retriesByTurn.set(entry.turn_number, 0);
+            }
+        }
+    }
+    return retriesByTurn;
+}
+function isRetryEntry(entry) {
+    if (entry.type !== 'agent_result')
+        return false;
+    const text = `${entry.assistant_message ?? ''} ${entry.output ?? ''}`.toLowerCase();
+    return text.includes('retry') || text.includes('api_retry');
+}
+function recommendTimeoutMs(turnDetails) {
+    if (turnDetails.length <= 1) {
+        return (turnDetails[0]?.durationMs ?? 0) + 30_000;
+    }
+    const totalDurationMs = turnDetails.reduce((sum, detail) => sum + detail.durationMs, 0);
+    return Math.max(totalDurationMs + 200_000, 30_000);
+}
+function formatTimeoutRecommendation(recommendedTimeoutMs, currentTimeoutMs) {
+    const recommendedSeconds = Math.round(recommendedTimeoutMs / 1000);
+    if (currentTimeoutMs == null) {
+        return `Recommended timeout: ~${recommendedSeconds}s`;
+    }
+    if (currentTimeoutMs > recommendedTimeoutMs) {
+        return `Timeout could be reduced to ~${recommendedSeconds}s`;
+    }
+    if (currentTimeoutMs < recommendedTimeoutMs) {
+        return `Timeout should be increased to ~${recommendedSeconds}s`;
+    }
+    return `Timeout looks right at ~${recommendedSeconds}s`;
+}

package/dist/reporters/github-comment.d.ts

@@ -0,0 +1,67 @@
+/**
+ * Pure markdown formatting and GitHub PR comment posting for pathgrade.
+ *
+ * The pure formatter (`formatReportMarkdown`) has no side effects. The
+ * GitHub API layer (`resolvePrContext`, `postOrUpdateComment`) talks to
+ * the GitHub issues-comments API via `fetch`, with graceful fallback on
+ * missing context or HTTP errors — the `pathgrade report` command is
+ * display-only and must never fail CI.
+ */
+import type { PathgradeReport, PathgradeSelectionReport } from '../types.js';
+export interface FormatOptions {
+    commentId: string;
+}
+/** Minimal body posted when `.pathgrade/results.json` is missing. */
+export declare const MISSING_RESULTS_BODY = "Pathgrade evals did not produce results. Check the workflow logs.";
+export declare function commentMarker(commentId: string): string;
+/**
+ * Format a pathgrade PR comment as markdown.
+ *
+ * The string starts with the dedup comment marker and is intended to be
+ * posted as a GitHub issue comment body.
+ */
+export declare function formatReportMarkdown(report: PathgradeReport, opts: FormatOptions): string;
+/**
+ * Render the `### Selection` section for the PR comment. Pure: no fs,
+ * no network, no reporter side effects. Three variants:
+ *
+ *   - `global_match` set → "global trigger fired" one-liner, no collapsible.
+ *   - `skipped` empty + `selected` non-empty → "every eval had a matching
+ *     change" one-liner.
+ *   - otherwise → "Ran **X of Y**" line + `<details>` block listing
+ *     skipped eval paths as inline code.
+ */
+export declare function formatSelectionSection(selection: PathgradeSelectionReport): string;
+export interface PrContext {
+    owner: string;
+    repo: string;
+    prNumber: number;
+    token: string;
+}
+export interface PrEnv {
+    GITHUB_TOKEN?: string;
+    GITHUB_REPOSITORY?: string;
+    GITHUB_REF?: string;
+    GITHUB_EVENT_PATH?: string;
+}
+/**
+ * Resolve the GitHub PR context from CI env vars. Returns `null` when any
+ * required field is missing — callers should skip posting and fall back
+ * to stdout.
+ */
+export declare function resolvePrContext(env: PrEnv): PrContext | null;
+export interface PostOptions {
+    commentId: string;
+    /** Body WITHOUT the marker — the marker is prepended if not already present. */
+    body: string;
+}
+/**
+ * Find an existing PR comment carrying `<!-- pathgrade:${commentId} -->`
+ * and update it; otherwise create a new one. Swallows all errors (logs
+ * to stderr) — `pathgrade report` must never fail CI.
+ *
+ * Bodies longer than GitHub's 65,536-character limit are truncated with a
+ * sentinel pointing to workflow artifacts, preserving the leading dedup
+ * marker so subsequent runs still find and update this comment.
+ */
+export declare function postOrUpdateComment(ctx: PrContext, opts: PostOptions): Promise<void>;

package/dist/reporters/github-comment.js

@@ -0,0 +1,252 @@
+/**
+ * Pure markdown formatting and GitHub PR comment posting for pathgrade.
+ *
+ * The pure formatter (`formatReportMarkdown`) has no side effects. The
+ * GitHub API layer (`resolvePrContext`, `postOrUpdateComment`) talks to
+ * the GitHub issues-comments API via `fetch`, with graceful fallback on
+ * missing context or HTTP errors — the `pathgrade report` command is
+ * display-only and must never fail CI.
+ */
+import * as fs from 'fs';
+/** Minimal body posted when `.pathgrade/results.json` is missing. */
+export const MISSING_RESULTS_BODY = 'Pathgrade evals did not produce results. Check the workflow logs.';
+export function commentMarker(commentId) {
+    return `<!-- pathgrade:${commentId} -->`;
+}
+function pct(n) {
+    return `${(n * 100).toFixed(1)}%`;
+}
+function durationSeconds(ms) {
+    return `${(ms / 1000).toFixed(1)}s`;
+}
+function computeAvgDuration(trials) {
+    if (trials.length === 0)
+        return 0;
+    return trials.reduce((sum, t) => sum + (t.duration_ms ?? 0), 0) / trials.length;
+}
+function escapeTableCell(value) {
+    return value.replace(/\|/g, '\\|').replace(/\n/g, ' ');
+}
+/**
+ * Format a pathgrade PR comment as markdown.
+ *
+ * The string starts with the dedup comment marker and is intended to be
+ * posted as a GitHub issue comment body.
+ */
+export function formatReportMarkdown(report, opts) {
+    const totalTrials = report.groups.reduce((n, g) => n + g.trials.length, 0);
+    const p = report.overall_pass_rate;
+    const overallPassAtK = totalTrials > 0 ? 1 - Math.pow(1 - p, totalTrials) : 0;
+    const overallPassPowK = totalTrials > 0 ? Math.pow(p, totalTrials) : 0;
+    const icon = report.status === 'pass' ? '✅' : '❌';
+    const lines = [];
+    lines.push(commentMarker(opts.commentId));
+    lines.push('');
+    lines.push(`### ${icon} Pathgrade report`);
+    lines.push('');
+    lines.push(`**Pass rate:** ${pct(p)}  |  ` +
+        `**pass@${totalTrials}:** ${pct(overallPassAtK)}  |  ` +
+        `**pass^${totalTrials}:** ${pct(overallPassPowK)}`);
+    if (report.threshold != null) {
+        lines.push('');
+        lines.push(`Threshold: ${pct(report.threshold)} — ${report.status.toUpperCase()}`);
+    }
+    lines.push('');
+    lines.push('| Group | Pass rate | pass@k | pass^k | Skills | Avg duration |');
+    lines.push('|---|---|---|---|---|---|');
+    for (const group of report.groups) {
+        const skills = group.skills_used.length > 0 ? group.skills_used.join(', ') : '—';
+        const avg = computeAvgDuration(group.trials);
+        lines.push(`| ${escapeTableCell(group.task)} | ${pct(group.pass_rate)} | ${pct(group.pass_at_k)} | ${pct(group.pass_pow_k)} | ${escapeTableCell(skills)} | ${durationSeconds(avg)} |`);
+    }
+    if (report.selection) {
+        lines.push('');
+        lines.push(formatSelectionSection(report.selection));
+    }
+    for (const group of report.groups) {
+        lines.push('');
+        lines.push(formatGroupDetails(group));
+    }
+    lines.push('');
+    return lines.join('\n');
+}
+/**
+ * Render the `### Selection` section for the PR comment. Pure: no fs,
+ * no network, no reporter side effects. Three variants:
+ *
+ *   - `global_match` set → "global trigger fired" one-liner, no collapsible.
+ *   - `skipped` empty + `selected` non-empty → "every eval had a matching
+ *     change" one-liner.
+ *   - otherwise → "Ran **X of Y**" line + `<details>` block listing
+ *     skipped eval paths as inline code.
+ */
+export function formatSelectionSection(selection) {
+    const out = [];
+    out.push('### Selection');
+    const total = selection.selected.length + selection.skipped.length;
+    if (selection.global_match) {
+        out.push(`Ran **all ${total}** evals — global trigger \`${selection.global_match}\` matched.`);
+        return out.join('\n');
+    }
+    if (selection.skipped.length === 0 && selection.selected.length > 0) {
+        out.push(`Ran **all ${total}** evals — every eval had a matching change.`);
+        return out.join('\n');
+    }
+    out.push(`Ran **${selection.selected.length} of ${total}** evals based on changes vs \`${selection.base_ref}\`.`);
+    if (selection.skipped.length > 0) {
+        out.push('');
+        out.push(`<details><summary>${selection.skipped.length} skipped (unaffected)</summary>`);
+        out.push('');
+        for (const s of selection.skipped) {
+            out.push(`- \`${s.file}\``);
+        }
+        out.push('');
+        out.push('</details>');
+    }
+    return out.join('\n');
+}
+function parsePrNumberFromRef(ref) {
+    if (!ref)
+        return null;
+    // refs/pull/<n>/merge  or  refs/pull/<n>/head
+    const m = ref.match(/^refs\/pull\/(\d+)\//);
+    if (!m)
+        return null;
+    const n = parseInt(m[1], 10);
+    return Number.isFinite(n) ? n : null;
+}
+function readPrNumberFromEvent(eventPath) {
+    if (!eventPath)
+        return null;
+    try {
+        const raw = fs.readFileSync(eventPath, 'utf-8');
+        const payload = JSON.parse(raw);
+        const n = payload?.pull_request?.number;
+        return typeof n === 'number' && Number.isFinite(n) ? n : null;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Resolve the GitHub PR context from CI env vars. Returns `null` when any
+ * required field is missing — callers should skip posting and fall back
+ * to stdout.
+ */
+export function resolvePrContext(env) {
+    const token = env.GITHUB_TOKEN;
+    const repo = env.GITHUB_REPOSITORY;
+    if (!token || !repo)
+        return null;
+    const [owner, repoName] = repo.split('/', 2);
+    if (!owner || !repoName)
+        return null;
+    const prNumber = readPrNumberFromEvent(env.GITHUB_EVENT_PATH) ??
+        parsePrNumberFromRef(env.GITHUB_REF);
+    if (prNumber == null)
+        return null;
+    return { owner, repo: repoName, prNumber, token };
+}
+function apiHeaders(token) {
+    return {
+        Authorization: `Bearer ${token}`,
+        Accept: 'application/vnd.github+json',
+        'Content-Type': 'application/json',
+        'User-Agent': 'pathgrade-report',
+    };
+}
+async function listPrComments(ctx) {
+    const url = `https://api.github.com/repos/${ctx.owner}/${ctx.repo}/issues/${ctx.prNumber}/comments?per_page=100`;
+    const res = await fetch(url, { headers: apiHeaders(ctx.token) });
+    if (!res.ok) {
+        throw new Error(`GET ${url} returned ${res.status}`);
+    }
+    const data = (await res.json());
+    return Array.isArray(data) ? data : [];
+}
+async function createPrComment(ctx, body) {
+    const url = `https://api.github.com/repos/${ctx.owner}/${ctx.repo}/issues/${ctx.prNumber}/comments`;
+    const res = await fetch(url, {
+        method: 'POST',
+        headers: apiHeaders(ctx.token),
+        body: JSON.stringify({ body }),
+    });
+    if (!res.ok) {
+        throw new Error(`POST ${url} returned ${res.status}`);
+    }
+}
+async function updatePrComment(ctx, commentId, body) {
+    const url = `https://api.github.com/repos/${ctx.owner}/${ctx.repo}/issues/comments/${commentId}`;
+    const res = await fetch(url, {
+        method: 'PATCH',
+        headers: apiHeaders(ctx.token),
+        body: JSON.stringify({ body }),
+    });
+    if (!res.ok) {
+        throw new Error(`PATCH ${url} returned ${res.status}`);
+    }
+}
+/** GitHub's documented hard limit for issue-comment bodies. */
+const GITHUB_COMMENT_MAX = 65536;
+const TRUNCATION_SENTINEL = '\n\n_…output truncated to fit GitHub\'s comment size limit — see the workflow artifacts for the full report._';
+/**
+ * Find an existing PR comment carrying `<!-- pathgrade:${commentId} -->`
+ * and update it; otherwise create a new one. Swallows all errors (logs
+ * to stderr) — `pathgrade report` must never fail CI.
+ *
+ * Bodies longer than GitHub's 65,536-character limit are truncated with a
+ * sentinel pointing to workflow artifacts, preserving the leading dedup
+ * marker so subsequent runs still find and update this comment.
+ */
+export async function postOrUpdateComment(ctx, opts) {
+    const marker = commentMarker(opts.commentId);
+    const withMarker = opts.body.startsWith(marker) ? opts.body : `${marker}\n${opts.body}`;
+    const body = withMarker.length <= GITHUB_COMMENT_MAX
+        ? withMarker
+        : withMarker.slice(0, GITHUB_COMMENT_MAX - TRUNCATION_SENTINEL.length) +
+            TRUNCATION_SENTINEL;
+    try {
+        const existing = await listPrComments(ctx);
+        const match = existing.find((c) => typeof c.body === 'string' && c.body.includes(marker));
+        if (match) {
+            await updatePrComment(ctx, match.id, body);
+        }
+        else {
+            await createPrComment(ctx, body);
+        }
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        console.error(`pathgrade report: failed to post PR comment — ${message}`);
+    }
+}
+function formatGroupDetails(group) {
+    const out = [];
+    out.push('<details>');
+    out.push(`<summary>${escapeTableCell(group.task)} — per-trial breakdown</summary>`);
+    out.push('');
+    for (const trial of group.trials) {
+        out.push(`#### ${trial.name ?? `trial ${trial.trial_id}`}`);
+        const reason = trial.diagnostics?.completionReason ?? '—';
+        out.push(`reward: **${trial.reward.toFixed(2)}**  |  duration: ${durationSeconds(trial.duration_ms)}  |  completion: \`${reason}\``);
+        if (trial.scorer_results.length > 0) {
+            out.push('');
+            out.push('| Scorer | Score | Weight | Details |');
+            out.push('|---|---|---|---|');
+            for (const s of trial.scorer_results) {
+                out.push(`| ${escapeTableCell(s.scorer_type)} | ${s.score.toFixed(2)} | ${s.weight.toFixed(2)} | ${escapeTableCell(s.details ?? '')} |`);
+            }
+        }
+        const warnings = trial.diagnostics?.warnings ?? [];
+        if (warnings.length > 0) {
+            out.push('');
+            out.push('Warnings:');
+            for (const w of warnings) {
+                out.push(`- ${w}`);
+            }
+        }
+        out.push('');
+    }
+    out.push('</details>');
+    return out.join('\n');
+}

package/dist/reporters/loader.d.ts

@@ -0,0 +1,8 @@
+import type { EvalReport } from '../types.js';
+export interface LoadedReport extends EvalReport {
+    file: string;
+    timestamp?: string;
+}
+export declare function loadReports(resultsDir: string, opts?: {
+    skipTraces?: boolean;
+}): Promise<LoadedReport[]>;