@wix/pathgrade 0.37.0 → 1.0.0

package/dist/plugin/reporter.js

@@ -1,280 +1 @@
-import * as path from 'path';
-import { execSync } from 'child_process';
-import { extractSkillsFromLog } from '../tool-events.js';
-import { fmt } from '../utils/cli.js';
-import { getPathgradeDir } from '../reporters/results-path.js';
-import fs from 'fs-extra';
-import { formatDiagnostics, formatDiagnosticsSummary } from '../reporters/diagnostics.js';
-import { readSidecar } from '../affected/sidecar.js';
-/**
- * Custom vitest reporter that layers pathgrade aggregate statistics
- * on top of vitest's default output.
- */
-export class PathgradeReporter {
-    opts;
-    constructor(opts) {
-        this.opts = opts ?? {};
-    }
-    async onTestRunEnd(testModules) {
-        const groups = this.collectGroups(testModules);
-        if (groups.length === 0)
-            return;
-        const mode = this.opts.reporter ?? 'cli';
-        if (mode === 'cli' || mode === 'browser') {
-            this.printCliSummary(groups);
-        }
-        await this.writeJsonResults(groups);
-        if (mode === 'browser') {
-            this.openBrowserViewer(groups);
-        }
-        // CI threshold check
-        if (this.opts.ci?.threshold != null) {
-            const allScores = groups.flatMap((g) => g.tests.map((t) => t.score));
-            if (allScores.length > 0) {
-                const avg = allScores.reduce((a, b) => a + b, 0) / allScores.length;
-                if (avg < this.opts.ci.threshold) {
-                    console.log(`\n  ${fmt.fail('CI THRESHOLD FAILED')}  avg score ${fmt.bold(avg.toFixed(3))} < threshold ${fmt.bold(String(this.opts.ci.threshold))}\n`);
-                    process.exitCode = 1;
-                }
-            }
-        }
-    }
-    collectGroups(testModules) {
-        const groupMap = new Map();
-        for (const mod of testModules) {
-            for (const testCase of mod.children.allTests()) {
-                const groupKey = this.getGroupKey(testCase);
-                const entry = this.toTestEntry(testCase);
-                if (!this.isReportableEntry(entry))
-                    continue;
-                if (!groupMap.has(groupKey)) {
-                    groupMap.set(groupKey, []);
-                }
-                groupMap.get(groupKey).push(entry);
-            }
-        }
-        return Array.from(groupMap.entries()).map(([groupName, tests]) => ({
-            groupName,
-            tests,
-        }));
-    }
-    /**
-     * Group key is the parent suite's full name (file + describe hierarchy).
-     * Tests without a describe wrapper are grouped by module path.
-     */
-    getGroupKey(testCase) {
-        const parent = testCase.parent;
-        const modulePath = testCase.module.relativeModuleId;
-        if (parent.type === 'suite') {
-            return `${modulePath} > ${parent.fullName}`;
-        }
-        return modulePath;
-    }
-    toTestEntry(testCase) {
-        const meta = testCase.meta();
-        const pathgradeMeta = meta.pathgrade;
-        const diag = testCase.diagnostic();
-        const result = testCase.result();
-        let score = 0;
-        let trial;
-        let diagnostics;
-        if (pathgradeMeta === undefined) {
-            // Test didn't use pathgrade — infer from test pass/fail
-            score = result.state === 'passed' ? 1 : 0;
-        }
-        else if (pathgradeMeta.length === 0) {
-            // Anomaly: pathgrade was set up but evaluate() was never called
-            console.warn(`  [pathgrade] warning: empty results for "${testCase.name}" — evaluate() may not have been called`);
-            score = result.state === 'passed' ? 1 : 0;
-        }
-        else {
-            // Use the last eval result's score
-            score = pathgradeMeta[pathgradeMeta.length - 1].score;
-            trial = pathgradeMeta[pathgradeMeta.length - 1].trial;
-            diagnostics = pathgradeMeta[pathgradeMeta.length - 1].diagnostics;
-        }
-        return {
-            name: testCase.name,
-            score,
-            durationMs: diag?.duration ?? 0,
-            state: result.state === 'pending' ? 'pending' : result.state,
-            trial,
-            diagnostics,
-        };
-    }
-    isReportableEntry(entry) {
-        return entry.state !== 'skipped' && entry.state !== 'pending';
-    }
-    printCliSummary(groups) {
-        console.log(`\n${fmt.bold('── pathgrade summary ')}${fmt.dim('─'.repeat(40))}\n`);
-        const forceVerbose = this.opts.diagnostics === true || process.env.PATHGRADE_DIAGNOSTICS === '1';
-        const currentTimeoutMs = this.opts.timeout != null ? this.opts.timeout * 1000 : undefined;
-        for (const group of groups) {
-            const k = group.tests.length;
-            const passRate = this.computePassRate(group.tests);
-            const passAtK = this.computePassAtK(passRate, k);
-            const passPowK = this.computePassPowK(passRate, k);
-            const avgDuration = this.computeAvgDuration(group.tests);
-            const prColor = passRate >= 0.5 ? fmt.green : fmt.red;
-            console.log(`  ${fmt.bold(group.groupName)}`);
-            console.log(`    ${fmt.dim('pass rate'.padEnd(12))} ${prColor((passRate * 100).toFixed(1) + '%')}`);
-            console.log(`    ${fmt.dim(`pass@${k}`.padEnd(12))} ${(passAtK * 100).toFixed(1)}%`);
-            console.log(`    ${fmt.dim(`pass^${k}`.padEnd(12))} ${(passPowK * 100).toFixed(1)}%`);
-            console.log(`    ${fmt.dim('avg time'.padEnd(12))} ${(avgDuration / 1000).toFixed(1)}s`);
-            console.log(`    ${fmt.dim('trials'.padEnd(12))} ${k}`);
-            console.log();
-            for (const test of group.tests) {
-                if (!test.diagnostics)
-                    continue;
-                const shouldPrintFull = forceVerbose || test.state !== 'passed' || test.diagnostics.completionReason === 'timeout' || test.diagnostics.completionReason === 'agent_crashed';
-                console.log(`    ${fmt.bold(test.name)}`);
-                const formatted = shouldPrintFull
-                    ? formatDiagnostics(test.diagnostics, { verbose: true, currentTimeoutMs })
-                    : formatDiagnosticsSummary(test.diagnostics);
-                for (const line of formatted.split('\n')) {
-                    console.log(`      ${line}`);
-                }
-                console.log();
-            }
-        }
-    }
-    async writeJsonResults(groups) {
-        const outputDir = getPathgradeDir(process.cwd());
-        const tracesDir = path.join(outputDir, 'traces');
-        await fs.ensureDir(tracesDir);
-        // Auto-create .gitignore if it doesn't exist
-        const gitignorePath = path.join(outputDir, '.gitignore');
-        if (!(await fs.pathExists(gitignorePath))) {
-            await fs.writeFile(gitignorePath, '*\n');
-        }
-        const allScores = groups.flatMap((g) => g.tests.map((t) => t.score));
-        const overallPassRate = allScores.length > 0
-            ? allScores.reduce((a, b) => a + b, 0) / allScores.length
-            : 0;
-        const threshold = this.opts.ci?.threshold;
-        const status = threshold != null
-            ? (overallPassRate >= threshold ? 'pass' : 'fail')
-            : (groups.every(g => g.tests.every(t => t.state === 'passed')) ? 'pass' : 'fail');
-        const consolidatedGroups = [];
-        for (const group of groups) {
-            const report = this.buildEvalReport(group);
-            const slug = this.slug(group.groupName);
-            const traceFile = `traces/${slug}.json`;
-            // Write full trace data (with session_log and conversation)
-            await fs.writeJson(path.join(outputDir, traceFile), report.trials, { spaces: 2 });
-            // Strip session_log and conversation from trials in consolidated report
-            const strippedTrials = report.trials.map(t => {
-                const { session_log, conversation, ...rest } = t;
-                return rest;
-            });
-            const { trials: _omitted, ...rest } = report;
-            consolidatedGroups.push({
-                ...rest,
-                trials: strippedTrials,
-                trace_file: traceFile,
-            });
-        }
-        // Merge selection metadata from the CLI-written sidecar, if any.
-        // Missing sidecar → plain run; malformed → warn and proceed without it.
-        const selection = await readSidecar(process.cwd(), msg => {
-            console.warn(`[pathgrade] ${msg}`);
-        });
-        const consolidated = {
-            version: 1,
-            timestamp: new Date().toISOString(),
-            ...(threshold != null ? { threshold } : {}),
-            overall_pass_rate: overallPassRate,
-            status,
-            groups: consolidatedGroups,
-            ...(selection ? { selection } : {}),
-        };
-        await fs.writeJson(path.join(outputDir, 'results.json'), consolidated, { spaces: 2 });
-        console.log(`\n  ${fmt.dim('Results written to')} ${outputDir}\n`);
-    }
-    buildEvalReport(group) {
-        const trials = group.tests.map((test, index) => this.toTrialResult(test, index));
-        const passRate = this.computePassRate(group.tests);
-        // Aggregate skills_used across all trials (deduplicated)
-        const allSkills = new Set();
-        for (const trial of trials) {
-            for (const skill of trial.skills_used ?? []) {
-                allSkills.add(skill);
-            }
-        }
-        return {
-            task: group.groupName,
-            pass_rate: passRate,
-            pass_at_k: this.computePassAtK(passRate, trials.length),
-            pass_pow_k: this.computePassPowK(passRate, trials.length),
-            trials,
-            skills_used: [...allSkills],
-        };
-    }
-    toTrialResult(test, index) {
-        const base = test.trial ?? {
-            trial_id: index + 1,
-            reward: test.score,
-            scorer_results: [],
-            duration_ms: test.durationMs,
-            n_commands: 0,
-            input_tokens: 0,
-            output_tokens: 0,
-            session_log: [],
-        };
-        // Extract skills_used from session_log tool events if not already present
-        const skills = base.skills_used ?? extractSkillsFromLog(base.session_log);
-        return {
-            ...base,
-            trial_id: index + 1,
-            name: test.name,
-            duration_ms: base.duration_ms || test.durationMs,
-            diagnostics: test.diagnostics ?? base.diagnostics,
-            ...(skills.length > 0 ? { skills_used: skills } : {}),
-        };
-    }
-    slug(value) {
-        return value
-            .toLowerCase()
-            .replace(/[^a-z0-9]+/g, '-')
-            .replace(/^-+|-+$/g, '') || 'report';
-    }
-    openBrowserViewer(_groups) {
-        // Write results JSON first, then open the viewer
-        const viewerPath = path.resolve(import.meta.dirname, '..', 'viewer.html');
-        try {
-            const openCmd = process.platform === 'darwin' ? 'open'
-                : process.platform === 'win32' ? 'start'
-                    : 'xdg-open';
-            execSync(`${openCmd} "${viewerPath}"`, { stdio: 'ignore' });
-            console.log(`\n  ${fmt.dim('Opened viewer in browser')}\n`);
-        }
-        catch {
-            console.log(`\n  ${fmt.dim('Open manually:')} ${viewerPath}\n`);
-        }
-    }
-    computePassRate(tests) {
-        if (tests.length === 0)
-            return 0;
-        const passed = tests.filter((t) => t.state === 'passed').length;
-        return passed / tests.length;
-    }
-    /**
-     * pass@k = 1 - (1 - p)^k
-     * Probability of at least 1 success in k trials.
-     */
-    computePassAtK(passRate, k) {
-        return 1 - Math.pow(1 - passRate, k);
-    }
-    /**
-     * pass^k = p^k
-     * Probability of all k trials succeeding.
-     */
-    computePassPowK(passRate, k) {
-        return Math.pow(passRate, k);
-    }
-    computeAvgDuration(tests) {
-        if (tests.length === 0)
-            return 0;
-        return tests.reduce((sum, t) => sum + t.durationMs, 0) / tests.length;
-    }
-}
+export { PathgradeReporter } from '../adapters/vitest/reporter.js';

package/dist/plugin/setup.d.ts

@@ -1 +1 @@
-export {};
+import '../adapters/vitest/setup.js';

package/dist/plugin/setup.js

@@ -1,3 +1 @@
-import { afterAll, afterEach } from 'vitest';
-import { lifecycle } from './lifecycle.js';
-lifecycle.install(afterEach, afterAll);
+import '../adapters/vitest/setup.js';

package/dist/providers/copy-filter.d.ts

@@ -16,3 +16,8 @@ export declare const DEFAULT_COPY_IGNORE: readonly string[];
  *   Pass `[]` to disable filtering. Invalid glob patterns are silently ignored.
  */
 export declare function createCopyFilter(ignorePatterns: readonly string[]): (src: string) => boolean;
+/**
+ * fs-extra cannot copy sockets, FIFOs, or device files. Host config caches can
+ * contain transient sockets, so filter them out before copy() reaches them.
+ */
+export declare function isPortableCopyEntry(src: string): boolean;

package/dist/providers/copy-filter.js

@@ -5,6 +5,7 @@
  * when copying workspace/skill directories into the sandbox.
  */
 import * as path from 'path';
+import * as fs from 'fs';
 import picomatch from 'picomatch';
 /**
  * Default ignore patterns applied when copying workspace and skill directories.
@@ -64,3 +65,11 @@ export function createCopyFilter(ignorePatterns) {
         return true;
     };
 }
+/**
+ * fs-extra cannot copy sockets, FIFOs, or device files. Host config caches can
+ * contain transient sockets, so filter them out before copy() reaches them.
+ */
+export function isPortableCopyEntry(src) {
+    const stat = fs.lstatSync(src);
+    return stat.isFile() || stat.isDirectory() || stat.isSymbolicLink();
+}

package/dist/providers/sandbox.js

@@ -1,7 +1,7 @@
 import fs from 'fs-extra';
 import * as path from 'path';
 import * as os from 'os';
-import { DEFAULT_COPY_IGNORE, createCopyFilter } from './copy-filter.js';
+import { DEFAULT_COPY_IGNORE, createCopyFilter, isPortableCopyEntry } from './copy-filter.js';
 export const SAFE_HOST_VARS = [
     'PATH', 'SHELL', 'LANG', 'LC_ALL', 'LC_CTYPE', 'TERM', 'USER', 'LOGNAME',
 ];
@@ -56,7 +56,7 @@ export async function createSandbox(spec) {
             if (await fs.pathExists(srcPath)) {
                 const destPath = path.join(homePath, relPath);
                 await fs.ensureDir(path.dirname(destPath));
-                await fs.copy(srcPath, destPath);
+                await fs.copy(srcPath, destPath, { filter: isPortableCopyEntry });
             }
         }
     }

package/dist/providers/workspace.d.ts

@@ -11,5 +11,11 @@ export interface Workspace {
     }): Promise<CommandResult>;
     dispose(): Promise<void>;
 }
+interface RemoveSandboxRootOptions {
+    remove?: (target: string) => Promise<void>;
+    sleep?: (ms: number) => Promise<void>;
+    retryDelaysMs?: readonly number[];
+}
+export declare function removeSandboxRoot(rootDir: string, opts?: RemoveSandboxRootOptions): Promise<void>;
 export declare function linkPathsFromHostHome(pathsToLink: string[], sandboxHomePath: string): Promise<void>;
 export declare function prepareWorkspace(spec: SandboxConfig): Promise<Workspace>;

package/dist/providers/workspace.js

@@ -5,6 +5,38 @@ import { createSandbox } from './sandbox.js';
 import { stageMcpConfig } from './mcp-config.js';
 import { sandboxExec } from './sandbox-exec.js';
 import { resolveCredentials } from './credentials.js';
+import { isPortableCopyEntry } from './copy-filter.js';
+const SANDBOX_REMOVE_RETRY_DELAYS_MS = [
+    50,
+    100,
+    250,
+    500,
+    1_000,
+    2_000,
+    3_000,
+    5_000,
+];
+function isRetryableRemoveError(error) {
+    const code = error.code;
+    return code === 'ENOTEMPTY' || code === 'EBUSY' || code === 'EPERM';
+}
+export async function removeSandboxRoot(rootDir, opts = {}) {
+    const remove = opts.remove ?? ((target) => fs.remove(target));
+    const sleep = opts.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+    const retryDelaysMs = opts.retryDelaysMs ?? SANDBOX_REMOVE_RETRY_DELAYS_MS;
+    for (let attempt = 0;; attempt++) {
+        try {
+            await remove(rootDir);
+            return;
+        }
+        catch (error) {
+            if (!isRetryableRemoveError(error) || attempt >= retryDelaysMs.length) {
+                throw error;
+            }
+            await sleep(retryDelaysMs[attempt]);
+        }
+    }
+}
 async function copyPathsFromHostHome(pathsToCopy, sandboxHomePath) {
     const realHome = os.homedir();
     for (const relPath of pathsToCopy) {
@@ -13,7 +45,7 @@ async function copyPathsFromHostHome(pathsToCopy, sandboxHomePath) {
             continue;
         const destPath = path.join(sandboxHomePath, relPath);
         await fs.ensureDir(path.dirname(destPath));
-        await fs.copy(srcPath, destPath);
+        await fs.copy(srcPath, destPath, { filter: isPortableCopyEntry });
     }
 }
 export async function linkPathsFromHostHome(pathsToLink, sandboxHomePath) {
@@ -50,19 +82,7 @@ export async function prepareWorkspace(spec) {
                 if (disposed)
                     return;
                 disposed = true;
-                for (let attempt = 0; attempt < 4; attempt++) {
-                    try {
-                        await fs.remove(rootDir);
-                        return;
-                    }
-                    catch (error) {
-                        const code = error.code;
-                        const retryable = code === 'ENOTEMPTY' || code === 'EBUSY' || code === 'EPERM';
-                        if (!retryable || attempt === 3)
-                            throw error;
-                        await new Promise((r) => setTimeout(r, 50 * (attempt + 1)));
-                    }
-                }
+                await removeSandboxRoot(rootDir);
             },
         };
     }

package/dist/reporters/diagnostics.d.ts

@@ -1,46 +1,8 @@
-import type { LogEntry } from '../types.js';
-import type { ReactionFiredEntry, TurnDetail } from '../sdk/types.js';
-import { type RuntimePolicyAuditEntry } from '../sdk/runtime-policy.js';
-export interface DiagnosticsScorer {
-    name: string;
-    type: string;
-    score: number;
-    weight: number;
-    details?: string;
-    status?: 'ok' | 'error' | 'skipped';
-}
-export interface DiagnosticsTurnDetail extends TurnDetail {
-    apiRetries: number;
-}
-export type DiagnosticsRuntimePolicyEntry = RuntimePolicyAuditEntry;
-export interface DiagnosticsReport {
-    turns: number;
-    totalDurationMs: number;
-    completionReason?: string;
-    completionDetail?: string;
-    score?: number;
-    turnDetails: DiagnosticsTurnDetail[];
-    reactionsFired: ReactionFiredEntry[];
-    runtimePoliciesApplied: DiagnosticsRuntimePolicyEntry[];
-    recommendedTimeoutMs: number;
-    warnings: string[];
-    scorers: Array<DiagnosticsScorer & {
-        status: 'ok' | 'error' | 'skipped';
-    }>;
-}
-export interface BuildDiagnosticsReportInput {
-    completionReason?: string;
-    completionDetail?: string;
-    score?: number;
-    turnDetails?: TurnDetail[];
-    reactionsFired?: ReactionFiredEntry[];
-    scorers?: DiagnosticsScorer[];
-    log: LogEntry[];
-}
+import type { DiagnosticsReport } from '../sdk/diagnostics.js';
+export { buildDiagnosticsReport } from '../sdk/diagnostics.js';
 export interface FormatDiagnosticsOptions {
     verbose?: boolean;
     currentTimeoutMs?: number;
 }
-export declare function buildDiagnosticsReport(input: BuildDiagnosticsReportInput): DiagnosticsReport;
 export declare function formatDiagnosticsSummary(report: DiagnosticsReport): string;
 export declare function formatDiagnostics(report: DiagnosticsReport, opts?: FormatDiagnosticsOptions): string;

package/dist/reporters/diagnostics.js

@@ -1,35 +1,4 @@
-import { extractRuntimePolicyAuditEntries } from '../sdk/runtime-policy.js';
-export function buildDiagnosticsReport(input) {
-    const baseTurnDetails = (input.turnDetails && input.turnDetails.length > 0)
-        ? [...input.turnDetails]
-        : deriveTurnDetailsFromLog(input.log);
-    const retriesByTurn = countRetriesByTurn(input.log);
-    const turnDetails = baseTurnDetails
-        .sort((a, b) => a.turn - b.turn)
-        .map((detail) => ({
-        ...detail,
-        apiRetries: retriesByTurn.get(detail.turn) ?? 0,
-    }));
-    const warnings = turnDetails
-        .filter((detail) => detail.outputLines > 500)
-        .map((detail) => `Turn ${detail.turn}: output exceeded 500 lines (${detail.outputLines} lines)`);
-    return {
-        turns: turnDetails.length,
-        totalDurationMs: turnDetails.reduce((sum, detail) => sum + detail.durationMs, 0),
-        completionReason: input.completionReason,
-        completionDetail: input.completionDetail,
-        score: input.score,
-        turnDetails,
-        reactionsFired: input.reactionsFired ?? [],
-        runtimePoliciesApplied: extractRuntimePolicyAuditEntries(input.log),
-        recommendedTimeoutMs: recommendTimeoutMs(turnDetails),
-        warnings,
-        scorers: (input.scorers ?? []).map((scorer) => ({
-            ...scorer,
-            status: scorer.status ?? 'ok',
-        })),
-    };
-}
+export { buildDiagnosticsReport } from '../sdk/diagnostics.js';
 export function formatDiagnosticsSummary(report) {
     const turnsLabel = report.turns === 1 ? '1 turn' : `${report.turns} turns`;
     const durationLabel = `${(report.totalDurationMs / 1000).toFixed(1)}s`;
@@ -84,54 +53,6 @@ export function formatDiagnostics(report, opts = {}) {
     lines.push('', formatTimeoutRecommendation(report.recommendedTimeoutMs, opts.currentTimeoutMs));
     return lines.join('\n');
 }
-function deriveTurnDetailsFromLog(log) {
-    return log
-        .filter((entry) => entry.type === 'agent_result'
-        && entry.synthetic_blocked_prompt !== true
-        && typeof entry.turn_number === 'number'
-        && typeof entry.duration_ms === 'number'
-        && typeof entry.output_lines === 'number'
-        && typeof entry.output_chars === 'number')
-        .map((entry) => ({
-        turn: entry.turn_number,
-        durationMs: entry.duration_ms,
-        outputLines: entry.output_lines,
-        outputChars: entry.output_chars,
-    }));
-}
-function countRetriesByTurn(log) {
-    const retriesByTurn = new Map();
-    let pendingRetries = 0;
-    for (const entry of log) {
-        if (isRetryEntry(entry)) {
-            pendingRetries++;
-            continue;
-        }
-        if (entry.type === 'agent_result' && typeof entry.turn_number === 'number') {
-            if (pendingRetries > 0) {
-                retriesByTurn.set(entry.turn_number, (retriesByTurn.get(entry.turn_number) ?? 0) + pendingRetries);
-                pendingRetries = 0;
-            }
-            if (!retriesByTurn.has(entry.turn_number)) {
-                retriesByTurn.set(entry.turn_number, 0);
-            }
-        }
-    }
-    return retriesByTurn;
-}
-function isRetryEntry(entry) {
-    if (entry.type !== 'agent_result')
-        return false;
-    const text = `${entry.assistant_message ?? ''} ${entry.output ?? ''}`.toLowerCase();
-    return text.includes('retry') || text.includes('api_retry');
-}
-function recommendTimeoutMs(turnDetails) {
-    if (turnDetails.length <= 1) {
-        return (turnDetails[0]?.durationMs ?? 0) + 30_000;
-    }
-    const totalDurationMs = turnDetails.reduce((sum, detail) => sum + detail.durationMs, 0);
-    return Math.max(totalDurationMs + 200_000, 30_000);
-}
 function formatTimeoutRecommendation(recommendedTimeoutMs, currentTimeoutMs) {
     const recommendedSeconds = Math.round(recommendedTimeoutMs / 1000);
     if (currentTimeoutMs == null) {

package/dist/reporters/report-summary.d.ts

@@ -0,0 +1,6 @@
+import type { ReportSummaryGroup } from '../reporting/types.js';
+export interface PrintReportSummaryOptions {
+    forceVerbose?: boolean;
+    currentTimeoutMs?: number;
+}
+export declare function printReportSummary(groups: ReportSummaryGroup[], opts?: PrintReportSummaryOptions): void;

package/dist/reporters/report-summary.js

@@ -0,0 +1,29 @@
+import { fmt } from '../utils/cli.js';
+import { formatDiagnostics, formatDiagnosticsSummary } from './diagnostics.js';
+export function printReportSummary(groups, opts = {}) {
+    console.log(`\n${fmt.bold('── pathgrade summary ')}${fmt.dim('─'.repeat(40))}\n`);
+    for (const group of groups) {
+        const prColor = group.pass_rate >= 0.5 ? fmt.green : fmt.red;
+        console.log(`  ${fmt.bold(group.task)}`);
+        console.log(`    ${fmt.dim('pass rate'.padEnd(12))} ${prColor((group.pass_rate * 100).toFixed(1) + '%')}`);
+        console.log(`    ${fmt.dim(`pass@${group.trial_count}`.padEnd(12))} ${(group.pass_at_k * 100).toFixed(1)}%`);
+        console.log(`    ${fmt.dim(`pass^${group.trial_count}`.padEnd(12))} ${(group.pass_pow_k * 100).toFixed(1)}%`);
+        console.log(`    ${fmt.dim('avg time'.padEnd(12))} ${(group.average_duration_ms / 1000).toFixed(1)}s`);
+        console.log(`    ${fmt.dim('trials'.padEnd(12))} ${group.trial_count}`);
+        console.log();
+        for (const diagnostic of group.diagnostics) {
+            const shouldPrintFull = opts.forceVerbose
+                || diagnostic.state !== 'passed'
+                || diagnostic.report.completionReason === 'timeout'
+                || diagnostic.report.completionReason === 'agent_crashed';
+            console.log(`    ${fmt.bold(diagnostic.caseName)}`);
+            const formatted = shouldPrintFull
+                ? formatDiagnostics(diagnostic.report, { verbose: true, currentTimeoutMs: opts.currentTimeoutMs })
+                : formatDiagnosticsSummary(diagnostic.report);
+            for (const line of formatted.split('\n')) {
+                console.log(`      ${line}`);
+            }
+            console.log();
+        }
+    }
+}

package/dist/reporting/artifacts.d.ts

@@ -0,0 +1,2 @@
+import type { ArtifactWriteResult, PathgradeReportBuildResult } from './types.js';
+export declare function writePathgradeArtifacts(artifactRoot: string, built: PathgradeReportBuildResult): Promise<ArtifactWriteResult>;

package/dist/reporting/artifacts.js

@@ -0,0 +1,20 @@
+import path from 'node:path';
+import fs from 'fs-extra';
+export async function writePathgradeArtifacts(artifactRoot, built) {
+    await fs.ensureDir(path.join(artifactRoot, 'traces'));
+    const gitignorePath = path.join(artifactRoot, '.gitignore');
+    if (!(await fs.pathExists(gitignorePath))) {
+        await fs.writeFile(gitignorePath, '*\n');
+    }
+    const traceFiles = [];
+    for (const trace of built.traces) {
+        await fs.writeJson(path.join(artifactRoot, trace.traceFile), trace.trials, { spaces: 2 });
+        traceFiles.push(trace.traceFile);
+    }
+    const resultsPath = path.join(artifactRoot, 'results.json');
+    await fs.writeJson(resultsPath, built.report, { spaces: 2 });
+    return {
+        resultsPath,
+        traceFiles,
+    };
+}

package/dist/reporting/core.d.ts

@@ -0,0 +1,2 @@
+import type { PathgradeReportBuildResult, ReportRunInput } from './types.js';
+export declare function buildPathgradeReport(input: ReportRunInput): PathgradeReportBuildResult;