@wix/pathgrade 1.0.27 → 1.0.29

package/dist/internal/direct-mcp-v2/public-scenario-runtime.js

@@ -8,7 +8,7 @@ import { projectEndpoints } from './projection.js';
 import { admitRuntimeLock } from './runtime-lock.js';
 import { startScenarioHttpHost } from './scenario-http-host.js';
 import { createClaudeParentBridges } from './claude-direct-mcp.js';
-import { CLAUDE_DIRECT_MCP_DARWIN_ARM64_LOCK, CLAUDE_DIRECT_MCP_NORMALIZED_HANDSHAKE, resolveClaudeScenarioExecutable } from './claude-profile.js';
+import { observeClaudeRuntimeControlPlane, resolveClaudeScenarioExecutable, selectClaudeRuntimeTarget } from './claude-profile.js';
 const defaultCoordinatorDependencies = {
     startHost: startScenarioHttpHost,
     scanBearerSurfaces,
@@ -286,8 +286,10 @@ function scenarioScanRoots(workspacePath, environment) {
         ].filter((path) => typeof path === 'string' && path.length > 0))];
 }
 export async function startPublicClaudeScenarioRuntime(options, dependencies = {}) {
-    const lock = dependencies.lock ?? CLAUDE_DIRECT_MCP_DARWIN_ARM64_LOCK;
-    const resolveExecutable = dependencies.resolveExecutable ?? resolveClaudeScenarioExecutable;
+    const target = dependencies.lock ? undefined : selectClaudeRuntimeTarget();
+    const lock = dependencies.lock ?? target.lock;
+    const resolveExecutable = dependencies.resolveExecutable
+        ?? (async () => resolveClaudeScenarioExecutable(target));
     const coordinator = await startScenarioCoordinator({
         ...options,
         provider: 'claude',
@@ -301,16 +303,10 @@ export async function startPublicClaudeScenarioRuntime(options, dependencies = {
                 environment: options.environment,
                 forbiddenSecrets: [bearer.slice('Bearer '.length), bearer],
                 ...(dependencies.commandVersion ? { commandVersion: dependencies.commandVersion } : {}),
-                initializeControlPlane: async () => {
-                    if (typeof createSdkMcpServer !== 'function' || typeof sdkQuery !== 'function') {
-                        throw new Error('Claude SDK parent bridge methods unavailable');
-                    }
-                    return {
-                        normalizedHandshake: CLAUDE_DIRECT_MCP_NORMALIZED_HANDSHAKE,
-                        capabilities: lock.capabilities,
-                        close: async () => undefined,
-                    };
-                },
+                initializeControlPlane: async () => observeClaudeRuntimeControlPlane({
+                    createSdkMcpServer,
+                    query: sdkQuery,
+                }),
             });
             const projected = projectEndpoints(context.endpoints, admission.profile);
             if (!Array.isArray(projected))
@@ -320,7 +316,7 @@ export async function startPublicClaudeScenarioRuntime(options, dependencies = {
             }));
             const bridges = createClaudeParentBridges({ artifact: options.artifact, projections: materialized });
             return {
-                value: bridges,
+                value: { bridges, commandPath: admission.commandPath },
                 async interrupt() { },
                 async waitForTerminal() { },
                 async close() {
@@ -340,9 +336,10 @@ export async function startPublicClaudeScenarioRuntime(options, dependencies = {
         },
     });
     return {
-        claudeServers: coordinator.attachment,
+        claudeServers: coordinator.attachment.bridges,
         configureSession(sessionOptions) {
-            sessionOptions.scenarioMcpServers = coordinator.attachment;
+            sessionOptions.scenarioMcpServers = coordinator.attachment.bridges;
+            sessionOptions.admittedClaudeCodeExecutable = coordinator.attachment.commandPath;
         },
         beginTurn: coordinator.beginTurn,
         settleTurn: coordinator.settleTurn,

package/dist/pathgrade.js

@@ -201,7 +201,14 @@ export async function runPathgradeCli(options = {}) {
         await clearSidecar(process.cwd());
         const env = buildRunnerEnv(parsed);
         try {
-            const config = await resolvePathgradeConfig({ cwd: process.cwd() });
+            const config = await resolvePathgradeConfig({
+                cwd: process.cwd(),
+                cli: parsed.attempts === undefined ? undefined : { attempts: parsed.attempts },
+                runnerArgs: parsed.runnerArgs,
+            });
+            if (!parsed.quiet && config.attempts > 1) {
+                console.error(`pathgrade: ${config.attempts} sequential attempts per selected case`);
+            }
             const runner = await loadRunnerInvocationAdapter({
                 adapterName: parsed.adapterName ?? config.runner.adapter,
                 cwd: process.cwd(),
@@ -210,7 +217,7 @@ export async function runPathgradeCli(options = {}) {
             process.exitCode = await runner.run({
                 cwd: process.cwd(),
                 runnerArgs: [...config.runner.args, ...parsed.runnerArgs],
-                env,
+                env: { ...env, PATHGRADE_ATTEMPT_COUNT: String(config.attempts) },
             });
         }
         catch (err) {
@@ -233,6 +240,7 @@ function printHelp(cliName) {
                      [--since=<ref>]           Override base ref (implies git mode)
                      [--changed-files=<path>]  Use an explicit newline-delimited file list
                      [--adapter=<name|path>]   Select built-in or third-party runner adapter
+                     [--attempts=N]            Run every selected case N times sequentially
                      [--diagnostics]           Print full diagnostics for passing evals too
                      [--quiet]                 Suppress the run-start summary
                      [--verbose|-v]            Stream live per-turn events to stderr during the run

package/dist/reporters/cli.js

@@ -18,8 +18,8 @@ export async function runCliPreview(resultsDir, opts) {
     }
     console.log(`\n${fmt.bold('pathgrade preview')}  ${fmt.dim(`${entries.length} reports from ${resolved}`)}\n`);
     for (const { file, ...report } of entries) {
-        const passRate = report.pass_rate ?? 0;
-        const isPass = passRate >= 0.5;
+        const meanReward = report.mean_reward ?? report.pass_rate ?? 0;
+        const isPass = report.status === undefined ? meanReward >= 0.5 : report.status === 'pass';
         const trials = report.trials || [];
         const avgDur = trials.reduce((s, t) => s + (t.duration_ms || 0), 0) / (trials.length || 1);
         const totalTokens = trials.reduce((s, t) => s + (t.input_tokens || 0) + (t.output_tokens || 0) + (t.conversation_input_tokens || 0) + (t.conversation_output_tokens || 0), 0);
@@ -33,9 +33,9 @@ export async function runCliPreview(resultsDir, opts) {
         console.log();
         // ── Summary metrics
         const metrics = [
-            ['Pass Rate', `${(passRate * 100).toFixed(1)}%`],
-            ['pass@k', report.pass_at_k != null ? `${(report.pass_at_k * 100).toFixed(1)}%` : '—'],
-            ['pass^k', report.pass_pow_k != null ? `${(report.pass_pow_k * 100).toFixed(1)}%` : '—'],
+            ['Mean Reward', `${(meanReward * 100).toFixed(1)}%`],
+            ['Success Rate', report.success_rate != null ? `${(report.success_rate * 100).toFixed(1)}%` : '—'],
+            ['pass@k', formatPassAtK(report.pass_at_k, report.pass_at_k_unavailable_reason)],
             ['Avg Duration', `${(avgDur / 1000).toFixed(1)}s`],
             ['Total Tokens', `~${totalTokens}`],
             ['Skills', report.skills_used?.join(', ') || 'none'],
@@ -47,7 +47,7 @@ export async function runCliPreview(resultsDir, opts) {
         // ── Trials
         for (const trial of trials) {
             const evaluated = trial.reward !== undefined;
-            const tp = evaluated && trial.reward >= 0.5;
+            const tp = evaluated && (trial.runner_outcome === undefined ? trial.reward >= 0.5 : trial.runner_outcome === 'passed' && trial.reward === 1);
             const trialStatus = !evaluated ? fmt.dim('N/A') : tp ? fmt.pass('PASS') : fmt.fail('FAIL');
             const reward = fmt.bold(evaluated ? trial.reward.toFixed(2) : 'n/a');
             const dur = `${((trial.duration_ms || 0) / 1000).toFixed(1)}s`;
@@ -120,6 +120,13 @@ export async function runCliPreview(resultsDir, opts) {
         console.log();
     }
 }
+function formatPassAtK(value, reason) {
+    if (typeof value === 'number')
+        return `${(value * 100).toFixed(1)}% (legacy v1)`;
+    if (value)
+        return Object.entries(value).map(([k, metric]) => `@${k} ${(metric * 100).toFixed(1)}%`).join(', ');
+    return reason ? `— (${reason.replaceAll('_', ' ')})` : '—';
+}
 function formatScorerStatus(status) {
     switch (status) {
         case 'error':

package/dist/reporters/github-comment.js

@@ -37,6 +37,13 @@ export function commentMarker(commentId) {
 function pct(n) {
     return `${(n * 100).toFixed(1)}%`;
 }
+function formatPassAtK(value, reason) {
+    if (typeof value === 'number')
+        return `${pct(value)} (legacy v1)`;
+    if (value)
+        return Object.entries(value).map(([k, metric]) => `pass@${k} ${pct(metric)}`).join(', ');
+    return reason ? `Unavailable (${reason.replaceAll('_', ' ')})` : 'Unavailable';
+}
 function durationSeconds(ms) {
     return `${(ms / 1000).toFixed(1)}s`;
 }
@@ -55,10 +62,7 @@ function escapeTableCell(value) {
  * posted as a GitHub issue comment body.
  */
 export function formatReportMarkdown(report, opts) {
-    const totalTrials = report.groups.reduce((n, g) => n + g.trials.length, 0);
-    const p = report.overall_pass_rate;
-    const overallPassAtK = totalTrials > 0 ? 1 - Math.pow(1 - p, totalTrials) : 0;
-    const overallPassPowK = totalTrials > 0 ? Math.pow(p, totalTrials) : 0;
+    const meanReward = report.overall_mean_reward ?? report.overall_pass_rate;
     const icon = report.status === 'pass' ? '✅' : '❌';
     const lines = [];
     lines.push(commentMarker(opts.commentId));
@@ -67,9 +71,8 @@ export function formatReportMarkdown(report, opts) {
     lines.push('');
     lines.push(`**Last run:** \`${report.timestamp}\``);
     lines.push('');
-    lines.push(`**Pass rate:** ${pct(p)}  |  ` +
-        `**pass@${totalTrials}:** ${pct(overallPassAtK)}  |  ` +
-        `**pass^${totalTrials}:** ${pct(overallPassPowK)}`);
+    lines.push(`**Mean reward:** ${pct(meanReward)}` +
+        (report.attempts_requested ? `  |  **Attempts:** ${report.attempts_completed ?? 0}/${report.attempts_requested}` : ''));
     if (opts.detailsUrl) {
         lines.push('');
         lines.push(`**Details:** [Open the full report](${formatLinkDestination(opts.detailsUrl)})`);
@@ -83,12 +86,12 @@ export function formatReportMarkdown(report, opts) {
         lines.push(`Threshold: ${pct(report.threshold)} — ${report.status.toUpperCase()}`);
     }
     lines.push('');
-    lines.push('| Group | Pass rate | pass@k | pass^k | Skills | Avg duration |');
+    lines.push('| Group | Mean reward | Success rate | pass@k | Skills | Avg duration |');
     lines.push('|---|---|---|---|---|---|');
     for (const group of report.groups) {
         const skills = group.skills_used.length > 0 ? group.skills_used.join(', ') : '—';
         const avg = computeAvgDuration(group.trials);
-        lines.push(`| ${escapeTableCell(group.task)} | ${pct(group.pass_rate)} | ${pct(group.pass_at_k)} | ${pct(group.pass_pow_k)} | ${escapeTableCell(skills)} | ${durationSeconds(avg)} |`);
+        lines.push(`| ${escapeTableCell(group.task)} | ${pct(group.mean_reward ?? group.pass_rate ?? 0)} | ${group.success_rate === undefined ? '—' : pct(group.success_rate)} | ${escapeTableCell(formatPassAtK(group.pass_at_k, group.pass_at_k_unavailable_reason))} | ${escapeTableCell(skills)} | ${durationSeconds(avg)} |`);
     }
     if (report.selection) {
         lines.push('');

package/dist/reporters/loader.d.ts

@@ -2,6 +2,7 @@ import type { EvalReport } from '../types.js';
 export interface LoadedReport extends EvalReport {
     file: string;
     timestamp?: string;
+    status?: 'pass' | 'fail';
 }
 export declare function loadReports(resultsDir: string, opts?: {
     skipTraces?: boolean;

package/dist/reporters/loader.js

@@ -7,7 +7,13 @@ async function hydrateTraces(report, traceFile, resolved) {
     const traceTrials = await fs.readJSON(tracePath);
     if (!Array.isArray(traceTrials) || !Array.isArray(report.trials))
         return;
-    report.trials = report.trials.map((t, i) => ({ ...t, ...traceTrials[i] }));
+    const tracesByAttemptId = new Map(traceTrials
+        .filter((trial) => typeof trial?.attempt_id === 'string')
+        .map((trial) => [trial.attempt_id, trial]));
+    report.trials = report.trials.map((trial, index) => ({
+        ...trial,
+        ...(trial.attempt_id ? tracesByAttemptId.get(trial.attempt_id) : traceTrials[index]),
+    }));
 }
 export async function loadReports(resultsDir, opts) {
     const resolved = path.resolve(resultsDir);
@@ -25,8 +31,15 @@ export async function loadReports(resultsDir, opts) {
                 results.push({ file, ...raw });
                 continue;
             }
+            if (raw.version !== 1 && raw.version !== 2)
+                continue;
             for (const group of raw.groups) {
-                const report = { file, timestamp: raw.timestamp, ...group };
+                const report = {
+                    file,
+                    timestamp: raw.timestamp,
+                    ...group,
+                    status: groupStatus(raw, group),
+                };
                 if (!opts?.skipTraces && group.trace_file) {
                     await hydrateTraces(report, group.trace_file, resolved);
                 }
@@ -37,3 +50,15 @@ export async function loadReports(resultsDir, opts) {
     }
     return results;
 }
+function groupStatus(raw, group) {
+    if (group.status === 'pass' || group.status === 'fail')
+        return group.status;
+    const meanReward = group.mean_reward ?? group.pass_rate ?? 0;
+    if (typeof raw.threshold === 'number')
+        return meanReward >= raw.threshold ? 'pass' : 'fail';
+    const outcomes = Array.isArray(group.trials) ? group.trials.map((trial) => trial.runner_outcome) : [];
+    if (outcomes.length > 0 && outcomes.every((outcome) => (outcome === 'passed' || outcome === 'failed' || outcome === 'not-run'))) {
+        return outcomes.every((outcome) => outcome === 'passed') ? 'pass' : 'fail';
+    }
+    return meanReward >= 0.5 ? 'pass' : 'fail';
+}

package/dist/reporters/report-summary.js

@@ -3,13 +3,21 @@ import { formatDiagnostics, formatDiagnosticsSummary } from './diagnostics.js';
 export function printReportSummary(groups, opts = {}) {
     console.log(`\n${fmt.bold('── pathgrade summary ')}${fmt.dim('─'.repeat(40))}\n`);
     for (const group of groups) {
-        const prColor = group.pass_rate >= 0.5 ? fmt.green : fmt.red;
+        const rewardColor = group.mean_reward >= 0.5 ? fmt.green : fmt.red;
         console.log(`  ${fmt.bold(group.task)}`);
-        console.log(`    ${fmt.dim('pass rate'.padEnd(12))} ${prColor((group.pass_rate * 100).toFixed(1) + '%')}`);
-        console.log(`    ${fmt.dim(`pass@${group.trial_count}`.padEnd(12))} ${(group.pass_at_k * 100).toFixed(1)}%`);
-        console.log(`    ${fmt.dim(`pass^${group.trial_count}`.padEnd(12))} ${(group.pass_pow_k * 100).toFixed(1)}%`);
+        console.log(`    ${fmt.dim('mean reward'.padEnd(12))} ${rewardColor((group.mean_reward * 100).toFixed(1) + '%')}`);
+        console.log(`    ${fmt.dim('success'.padEnd(12))} ${group.success_rate === undefined ? '—' : `${(group.success_rate * 100).toFixed(1)}%`}`);
+        if (group.pass_at_k) {
+            for (const [k, value] of Object.entries(group.pass_at_k)) {
+                console.log(`    ${fmt.dim(`pass@${k}`.padEnd(12))} ${(value * 100).toFixed(1)}%`);
+            }
+        }
+        else {
+            const reason = group.pass_at_k_unavailable_reason?.replaceAll('_', ' ') ?? 'repeated binary attempts required';
+            console.log(`    ${fmt.dim('pass@k'.padEnd(12))} — (${reason})`);
+        }
         console.log(`    ${fmt.dim('avg time'.padEnd(12))} ${(group.average_duration_ms / 1000).toFixed(1)}s`);
-        console.log(`    ${fmt.dim('trials'.padEnd(12))} ${group.trial_count}`);
+        console.log(`    ${fmt.dim('attempts'.padEnd(12))} ${group.trial_count}`);
         console.log();
         for (const diagnostic of group.diagnostics) {
             const shouldPrintFull = opts.forceVerbose

package/dist/reporting/core.d.ts

@@ -1,2 +1,3 @@
 import type { PathgradeReportBuildResult, ReportRunInput } from './types.js';
 export declare function buildPathgradeReport(input: ReportRunInput): PathgradeReportBuildResult;
+export declare function finiteSamplePassAtK(n: number, successes: number, k: number): number;