@wix/pathgrade 1.0.27 → 1.0.28

package/dist/commands/report.js

@@ -43,10 +43,24 @@ function isPathgradeReport(value) {
     if (!value || typeof value !== 'object')
         return false;
     const v = value;
-    return (v.version === 1 &&
+    return ((v.version === 1 || v.version === 2) &&
         typeof v.overall_pass_rate === 'number' &&
         (v.status === 'pass' || v.status === 'fail') &&
-        Array.isArray(v.groups));
+        Array.isArray(v.groups) &&
+        (v.version === 1 || (typeof v.overall_mean_reward === 'number'
+            && hasValidAttemptCounts(v)
+            && v.groups.every(group => typeof group.mean_reward === 'number'))));
+}
+function hasValidAttemptCounts(report) {
+    const requested = report.attempts_requested;
+    const completed = report.attempts_completed;
+    return typeof requested === 'number'
+        && Number.isSafeInteger(requested)
+        && requested >= 1
+        && typeof completed === 'number'
+        && Number.isSafeInteger(completed)
+        && completed >= 0
+        && completed <= requested;
 }
 async function loadReport(resolvedPath) {
     if (!(await fs.pathExists(resolvedPath))) {

package/dist/commands/run-args.d.ts

@@ -8,6 +8,7 @@
 export interface PathgradeRunArgs {
     runnerArgs: string[];
     adapterName?: string;
+    attempts?: number;
     forceDiagnostics: boolean;
     forceVerbose: boolean;
     changed: boolean;

package/dist/commands/run-args.js

@@ -14,6 +14,7 @@ export function parsePathgradeRunArgs(args) {
     let since;
     let changedFilesPath;
     let adapterName;
+    let attempts;
     let passthrough = false;
     for (const arg of args) {
         if (passthrough) {
@@ -44,6 +45,10 @@ export function parsePathgradeRunArgs(args) {
             adapterName = arg.slice('--adapter='.length);
             continue;
         }
+        if (arg.startsWith('--attempts=')) {
+            attempts = parseAttempts(arg.slice('--attempts='.length));
+            continue;
+        }
         if (arg.startsWith('--since=')) {
             since = arg.slice('--since='.length);
             continue;
@@ -68,8 +73,16 @@ export function parsePathgradeRunArgs(args) {
         changed,
         quiet,
         adapterName,
+        attempts,
         since,
         changedFilesPath,
     };
     return warnings.length > 0 ? { ...base, warnings } : base;
 }
+function parseAttempts(value) {
+    const attempts = Number(value);
+    if (!Number.isSafeInteger(attempts) || attempts < 1) {
+        throw new Error('pathgrade: --attempts must be an integer greater than or equal to 1');
+    }
+    return attempts;
+}

package/dist/commands/run-changed.js

@@ -27,13 +27,13 @@ export async function runChanged(opts) {
     const runnerEnv = buildRunnerEnv(parsed, {
         PATHGRADE_SELECTION_INVOCATION_ID: selectionInvocationId,
     });
-    const configPath = findVitestConfigArg(parsed.runnerArgs);
     let config;
     let runnerInvocation;
     try {
         config = await resolvePathgradeConfig({
             cwd,
-            legacyVitestConfigPath: configPath,
+            cli: parsed.attempts === undefined ? undefined : { attempts: parsed.attempts },
+            runnerArgs: parsed.runnerArgs,
             warn: w => {
                 if (!parsed.quiet)
                     process.stderr.write(`${w}\n`);
@@ -107,6 +107,7 @@ export async function runChanged(opts) {
             totalEvals: evalFiles.length,
             changedCount: changedFiles.length,
             result,
+            attempts: config.attempts,
         });
     }
     // Persist the sidecar immediately — even on empty selection, so the
@@ -128,7 +129,7 @@ export async function runChanged(opts) {
         cwd,
         runnerArgs,
         selectedFiles,
-        env: runnerEnv,
+        env: { ...runnerEnv, PATHGRADE_ATTEMPT_COUNT: String(config.attempts) },
     });
 }
 function printRunStartSummary(input) {
@@ -140,6 +141,7 @@ function printRunStartSummary(input) {
     const globalLabel = result.globalMatch ? `\`${result.globalMatch}\`` : 'none';
     process.stderr.write(`          global matches: ${globalLabel}\n`);
     process.stderr.write(`          selected: ${result.selected.length} / ${totalEvals} evals\n`);
+    process.stderr.write(`          attempts: ${input.attempts} per selected case\n`);
     for (const entry of result.selected) {
         process.stderr.write(`            ${entry.file}\n`);
     }
@@ -158,15 +160,3 @@ function readChangedFilesList(filePath) {
 function errMsg(err) {
     return err instanceof Error ? err.message : String(err);
 }
-function findVitestConfigArg(args) {
-    for (let i = 0; i < args.length; i++) {
-        const arg = args[i];
-        if (arg === '--config' || arg === '-c')
-            return args[i + 1];
-        if (arg.startsWith('--config='))
-            return arg.slice('--config='.length);
-        if (arg.startsWith('-c='))
-            return arg.slice('-c='.length);
-    }
-    return undefined;
-}

package/dist/config/pathgrade.d.ts

@@ -1,4 +1,5 @@
 export interface PathgradeConfig {
+    attempts?: number;
     runner?: {
         adapter?: string;
         args?: string[];
@@ -18,6 +19,7 @@ export interface PathgradeConfig {
     };
 }
 export interface ResolvedPathgradeConfig {
+    attempts: number;
     runner: {
         adapter: string;
         args: string[];
@@ -44,5 +46,6 @@ export declare function resolvePathgradeConfig(input: {
     cli?: PathgradeConfig;
     configPath?: string;
     legacyVitestConfigPath?: string;
+    runnerArgs?: readonly string[];
     warn?: (message: string) => void;
 }): Promise<ResolvedPathgradeConfig>;

package/dist/config/pathgrade.js

@@ -17,6 +17,7 @@ const PATHGRADE_CONFIG_CANDIDATES = [
 ];
 export function defaultPathgradeConfig() {
     return {
+        attempts: 1,
         runner: {
             adapter: 'vitest',
             args: [],
@@ -35,13 +36,30 @@ export function defaultPathgradeConfig() {
 }
 export async function resolvePathgradeConfig(input) {
     const fileConfig = await loadPathgradeConfigFile(input.cwd, input.configPath);
-    const legacyConfig = await loadLegacyVitestConfig(input.cwd, input.legacyVitestConfigPath, input.warn);
+    const legacyConfig = await loadLegacyVitestConfig(input.cwd, input.legacyVitestConfigPath ?? findVitestConfigArg([
+        ...(fileConfig?.runner?.args ?? []),
+        ...(input.runnerArgs ?? []),
+    ]), input.warn);
     return mergePathgradeConfig(mergePathgradeConfig(mergePathgradeConfig(defaultPathgradeConfig(), legacyConfig), fileConfig), input.cli);
 }
+function findVitestConfigArg(args) {
+    let configPath;
+    for (let i = 0; i < args.length; i++) {
+        const arg = args[i];
+        if (arg === '--config' || arg === '-c')
+            configPath = args[i + 1];
+        else if (arg.startsWith('--config='))
+            configPath = arg.slice('--config='.length);
+        else if (arg.startsWith('-c='))
+            configPath = arg.slice('-c='.length);
+    }
+    return configPath;
+}
 function mergePathgradeConfig(base, override) {
     if (!override)
         return base;
     return {
+        attempts: override.attempts ?? base.attempts,
         runner: {
             adapter: override.runner?.adapter ?? base.runner.adapter,
             args: override.runner?.args ?? base.runner.args,
@@ -91,6 +109,7 @@ function validatePathgradeConfig(value, label) {
         throw invalidConfig(label, 'default export must be an object');
     }
     validateOptionalObject(value.runner, label, 'runner');
+    validateOptionalPositiveInteger(value.attempts, label, 'attempts');
     const runner = asOptionalObject(value.runner);
     validateOptionalString(runner?.adapter, label, 'runner.adapter');
     validateOptionalStringArray(runner?.args, label, 'runner.args');
@@ -139,6 +158,11 @@ function validateOptionalNumber(value, label, field) {
         throw invalidConfig(label, `${field} must be a number`);
     }
 }
+function validateOptionalPositiveInteger(value, label, field) {
+    if (value !== undefined && (!Number.isSafeInteger(value) || Number(value) < 1)) {
+        throw invalidConfig(label, `${field} must be an integer greater than or equal to 1`);
+    }
+}
 function invalidConfig(label, reason) {
     return new InvalidPathgradeConfigError(`pathgrade: invalid ${label}: ${reason}`);
 }
@@ -179,6 +203,14 @@ async function loadLegacyVitestConfig(cwd, configPath, warn = () => { }) {
     if (!isObject(opts))
         return undefined;
     return {
+        ...(opts.reporter === 'cli' || opts.reporter === 'browser' || opts.reporter === 'json'
+            ? { reporter: opts.reporter }
+            : {}),
+        ...(typeof opts.diagnostics === 'boolean' ? { diagnostics: opts.diagnostics } : {}),
+        ...(typeof opts.verbose === 'boolean' ? { verbose: opts.verbose } : {}),
+        ...(isObject(opts.ci) && typeof opts.ci.threshold === 'number'
+            ? { ci: { threshold: opts.ci.threshold } }
+            : {}),
         evals: {
             ...(Array.isArray(opts.include) ? { include: opts.include } : {}),
             ...(Array.isArray(opts.exclude) ? { exclude: opts.exclude } : {}),

package/dist/pathgrade.js

@@ -201,7 +201,14 @@ export async function runPathgradeCli(options = {}) {
         await clearSidecar(process.cwd());
         const env = buildRunnerEnv(parsed);
         try {
-            const config = await resolvePathgradeConfig({ cwd: process.cwd() });
+            const config = await resolvePathgradeConfig({
+                cwd: process.cwd(),
+                cli: parsed.attempts === undefined ? undefined : { attempts: parsed.attempts },
+                runnerArgs: parsed.runnerArgs,
+            });
+            if (!parsed.quiet && config.attempts > 1) {
+                console.error(`pathgrade: ${config.attempts} sequential attempts per selected case`);
+            }
             const runner = await loadRunnerInvocationAdapter({
                 adapterName: parsed.adapterName ?? config.runner.adapter,
                 cwd: process.cwd(),
@@ -210,7 +217,7 @@ export async function runPathgradeCli(options = {}) {
             process.exitCode = await runner.run({
                 cwd: process.cwd(),
                 runnerArgs: [...config.runner.args, ...parsed.runnerArgs],
-                env,
+                env: { ...env, PATHGRADE_ATTEMPT_COUNT: String(config.attempts) },
             });
         }
         catch (err) {
@@ -233,6 +240,7 @@ function printHelp(cliName) {
                      [--since=<ref>]           Override base ref (implies git mode)
                      [--changed-files=<path>]  Use an explicit newline-delimited file list
                      [--adapter=<name|path>]   Select built-in or third-party runner adapter
+                     [--attempts=N]            Run every selected case N times sequentially
                      [--diagnostics]           Print full diagnostics for passing evals too
                      [--quiet]                 Suppress the run-start summary
                      [--verbose|-v]            Stream live per-turn events to stderr during the run

package/dist/reporters/cli.js

@@ -18,8 +18,8 @@ export async function runCliPreview(resultsDir, opts) {
     }
     console.log(`\n${fmt.bold('pathgrade preview')}  ${fmt.dim(`${entries.length} reports from ${resolved}`)}\n`);
     for (const { file, ...report } of entries) {
-        const passRate = report.pass_rate ?? 0;
-        const isPass = passRate >= 0.5;
+        const meanReward = report.mean_reward ?? report.pass_rate ?? 0;
+        const isPass = report.status === undefined ? meanReward >= 0.5 : report.status === 'pass';
         const trials = report.trials || [];
         const avgDur = trials.reduce((s, t) => s + (t.duration_ms || 0), 0) / (trials.length || 1);
         const totalTokens = trials.reduce((s, t) => s + (t.input_tokens || 0) + (t.output_tokens || 0) + (t.conversation_input_tokens || 0) + (t.conversation_output_tokens || 0), 0);
@@ -33,9 +33,9 @@ export async function runCliPreview(resultsDir, opts) {
         console.log();
         // ── Summary metrics
         const metrics = [
-            ['Pass Rate', `${(passRate * 100).toFixed(1)}%`],
-            ['pass@k', report.pass_at_k != null ? `${(report.pass_at_k * 100).toFixed(1)}%` : '—'],
-            ['pass^k', report.pass_pow_k != null ? `${(report.pass_pow_k * 100).toFixed(1)}%` : '—'],
+            ['Mean Reward', `${(meanReward * 100).toFixed(1)}%`],
+            ['Success Rate', report.success_rate != null ? `${(report.success_rate * 100).toFixed(1)}%` : '—'],
+            ['pass@k', formatPassAtK(report.pass_at_k, report.pass_at_k_unavailable_reason)],
             ['Avg Duration', `${(avgDur / 1000).toFixed(1)}s`],
             ['Total Tokens', `~${totalTokens}`],
             ['Skills', report.skills_used?.join(', ') || 'none'],
@@ -47,7 +47,7 @@ export async function runCliPreview(resultsDir, opts) {
         // ── Trials
         for (const trial of trials) {
             const evaluated = trial.reward !== undefined;
-            const tp = evaluated && trial.reward >= 0.5;
+            const tp = evaluated && (trial.runner_outcome === undefined ? trial.reward >= 0.5 : trial.runner_outcome === 'passed' && trial.reward === 1);
             const trialStatus = !evaluated ? fmt.dim('N/A') : tp ? fmt.pass('PASS') : fmt.fail('FAIL');
             const reward = fmt.bold(evaluated ? trial.reward.toFixed(2) : 'n/a');
             const dur = `${((trial.duration_ms || 0) / 1000).toFixed(1)}s`;
@@ -120,6 +120,13 @@ export async function runCliPreview(resultsDir, opts) {
         console.log();
     }
 }
+function formatPassAtK(value, reason) {
+    if (typeof value === 'number')
+        return `${(value * 100).toFixed(1)}% (legacy v1)`;
+    if (value)
+        return Object.entries(value).map(([k, metric]) => `@${k} ${(metric * 100).toFixed(1)}%`).join(', ');
+    return reason ? `— (${reason.replaceAll('_', ' ')})` : '—';
+}
 function formatScorerStatus(status) {
     switch (status) {
         case 'error':

package/dist/reporters/github-comment.js

@@ -37,6 +37,13 @@ export function commentMarker(commentId) {
 function pct(n) {
     return `${(n * 100).toFixed(1)}%`;
 }
+function formatPassAtK(value, reason) {
+    if (typeof value === 'number')
+        return `${pct(value)} (legacy v1)`;
+    if (value)
+        return Object.entries(value).map(([k, metric]) => `pass@${k} ${pct(metric)}`).join(', ');
+    return reason ? `Unavailable (${reason.replaceAll('_', ' ')})` : 'Unavailable';
+}
 function durationSeconds(ms) {
     return `${(ms / 1000).toFixed(1)}s`;
 }
@@ -55,10 +62,7 @@ function escapeTableCell(value) {
  * posted as a GitHub issue comment body.
  */
 export function formatReportMarkdown(report, opts) {
-    const totalTrials = report.groups.reduce((n, g) => n + g.trials.length, 0);
-    const p = report.overall_pass_rate;
-    const overallPassAtK = totalTrials > 0 ? 1 - Math.pow(1 - p, totalTrials) : 0;
-    const overallPassPowK = totalTrials > 0 ? Math.pow(p, totalTrials) : 0;
+    const meanReward = report.overall_mean_reward ?? report.overall_pass_rate;
     const icon = report.status === 'pass' ? '✅' : '❌';
     const lines = [];
     lines.push(commentMarker(opts.commentId));
@@ -67,9 +71,8 @@ export function formatReportMarkdown(report, opts) {
     lines.push('');
     lines.push(`**Last run:** \`${report.timestamp}\``);
     lines.push('');
-    lines.push(`**Pass rate:** ${pct(p)}  |  ` +
-        `**pass@${totalTrials}:** ${pct(overallPassAtK)}  |  ` +
-        `**pass^${totalTrials}:** ${pct(overallPassPowK)}`);
+    lines.push(`**Mean reward:** ${pct(meanReward)}` +
+        (report.attempts_requested ? `  |  **Attempts:** ${report.attempts_completed ?? 0}/${report.attempts_requested}` : ''));
     if (opts.detailsUrl) {
         lines.push('');
         lines.push(`**Details:** [Open the full report](${formatLinkDestination(opts.detailsUrl)})`);
@@ -83,12 +86,12 @@ export function formatReportMarkdown(report, opts) {
         lines.push(`Threshold: ${pct(report.threshold)} — ${report.status.toUpperCase()}`);
     }
     lines.push('');
-    lines.push('| Group | Pass rate | pass@k | pass^k | Skills | Avg duration |');
+    lines.push('| Group | Mean reward | Success rate | pass@k | Skills | Avg duration |');
     lines.push('|---|---|---|---|---|---|');
     for (const group of report.groups) {
         const skills = group.skills_used.length > 0 ? group.skills_used.join(', ') : '—';
         const avg = computeAvgDuration(group.trials);
-        lines.push(`| ${escapeTableCell(group.task)} | ${pct(group.pass_rate)} | ${pct(group.pass_at_k)} | ${pct(group.pass_pow_k)} | ${escapeTableCell(skills)} | ${durationSeconds(avg)} |`);
+        lines.push(`| ${escapeTableCell(group.task)} | ${pct(group.mean_reward ?? group.pass_rate ?? 0)} | ${group.success_rate === undefined ? '—' : pct(group.success_rate)} | ${escapeTableCell(formatPassAtK(group.pass_at_k, group.pass_at_k_unavailable_reason))} | ${escapeTableCell(skills)} | ${durationSeconds(avg)} |`);
     }
     if (report.selection) {
         lines.push('');

package/dist/reporters/loader.d.ts

@@ -2,6 +2,7 @@ import type { EvalReport } from '../types.js';
 export interface LoadedReport extends EvalReport {
     file: string;
     timestamp?: string;
+    status?: 'pass' | 'fail';
 }
 export declare function loadReports(resultsDir: string, opts?: {
     skipTraces?: boolean;

package/dist/reporters/loader.js

@@ -7,7 +7,13 @@ async function hydrateTraces(report, traceFile, resolved) {
     const traceTrials = await fs.readJSON(tracePath);
     if (!Array.isArray(traceTrials) || !Array.isArray(report.trials))
         return;
-    report.trials = report.trials.map((t, i) => ({ ...t, ...traceTrials[i] }));
+    const tracesByAttemptId = new Map(traceTrials
+        .filter((trial) => typeof trial?.attempt_id === 'string')
+        .map((trial) => [trial.attempt_id, trial]));
+    report.trials = report.trials.map((trial, index) => ({
+        ...trial,
+        ...(trial.attempt_id ? tracesByAttemptId.get(trial.attempt_id) : traceTrials[index]),
+    }));
 }
 export async function loadReports(resultsDir, opts) {
     const resolved = path.resolve(resultsDir);
@@ -25,8 +31,15 @@ export async function loadReports(resultsDir, opts) {
                 results.push({ file, ...raw });
                 continue;
             }
+            if (raw.version !== 1 && raw.version !== 2)
+                continue;
             for (const group of raw.groups) {
-                const report = { file, timestamp: raw.timestamp, ...group };
+                const report = {
+                    file,
+                    timestamp: raw.timestamp,
+                    ...group,
+                    status: groupStatus(raw, group),
+                };
                 if (!opts?.skipTraces && group.trace_file) {
                     await hydrateTraces(report, group.trace_file, resolved);
                 }
@@ -37,3 +50,15 @@ export async function loadReports(resultsDir, opts) {
     }
     return results;
 }
+function groupStatus(raw, group) {
+    if (group.status === 'pass' || group.status === 'fail')
+        return group.status;
+    const meanReward = group.mean_reward ?? group.pass_rate ?? 0;
+    if (typeof raw.threshold === 'number')
+        return meanReward >= raw.threshold ? 'pass' : 'fail';
+    const outcomes = Array.isArray(group.trials) ? group.trials.map((trial) => trial.runner_outcome) : [];
+    if (outcomes.length > 0 && outcomes.every((outcome) => (outcome === 'passed' || outcome === 'failed' || outcome === 'not-run'))) {
+        return outcomes.every((outcome) => outcome === 'passed') ? 'pass' : 'fail';
+    }
+    return meanReward >= 0.5 ? 'pass' : 'fail';
+}

package/dist/reporters/report-summary.js

@@ -3,13 +3,21 @@ import { formatDiagnostics, formatDiagnosticsSummary } from './diagnostics.js';
 export function printReportSummary(groups, opts = {}) {
     console.log(`\n${fmt.bold('── pathgrade summary ')}${fmt.dim('─'.repeat(40))}\n`);
     for (const group of groups) {
-        const prColor = group.pass_rate >= 0.5 ? fmt.green : fmt.red;
+        const rewardColor = group.mean_reward >= 0.5 ? fmt.green : fmt.red;
         console.log(`  ${fmt.bold(group.task)}`);
-        console.log(`    ${fmt.dim('pass rate'.padEnd(12))} ${prColor((group.pass_rate * 100).toFixed(1) + '%')}`);
-        console.log(`    ${fmt.dim(`pass@${group.trial_count}`.padEnd(12))} ${(group.pass_at_k * 100).toFixed(1)}%`);
-        console.log(`    ${fmt.dim(`pass^${group.trial_count}`.padEnd(12))} ${(group.pass_pow_k * 100).toFixed(1)}%`);
+        console.log(`    ${fmt.dim('mean reward'.padEnd(12))} ${rewardColor((group.mean_reward * 100).toFixed(1) + '%')}`);
+        console.log(`    ${fmt.dim('success'.padEnd(12))} ${group.success_rate === undefined ? '—' : `${(group.success_rate * 100).toFixed(1)}%`}`);
+        if (group.pass_at_k) {
+            for (const [k, value] of Object.entries(group.pass_at_k)) {
+                console.log(`    ${fmt.dim(`pass@${k}`.padEnd(12))} ${(value * 100).toFixed(1)}%`);
+            }
+        }
+        else {
+            const reason = group.pass_at_k_unavailable_reason?.replaceAll('_', ' ') ?? 'repeated binary attempts required';
+            console.log(`    ${fmt.dim('pass@k'.padEnd(12))} — (${reason})`);
+        }
         console.log(`    ${fmt.dim('avg time'.padEnd(12))} ${(group.average_duration_ms / 1000).toFixed(1)}s`);
-        console.log(`    ${fmt.dim('trials'.padEnd(12))} ${group.trial_count}`);
+        console.log(`    ${fmt.dim('attempts'.padEnd(12))} ${group.trial_count}`);
         console.log();
         for (const diagnostic of group.diagnostics) {
             const shouldPrintFull = opts.forceVerbose

package/dist/reporting/core.d.ts

@@ -1,2 +1,3 @@
 import type { PathgradeReportBuildResult, ReportRunInput } from './types.js';
 export declare function buildPathgradeReport(input: ReportRunInput): PathgradeReportBuildResult;
+export declare function finiteSamplePassAtK(n: number, successes: number, k: number): number;