@wix/pathgrade 1.0.26 → 1.0.28

package/dist/agents/opencode/protocol.js

@@ -0,0 +1,47 @@
+export const NATIVE_TOOL_ACTIONS = {
+    bash: 'run_shell',
+    read: 'read_file',
+    write: 'write_file',
+    edit: 'edit_file',
+    grep: 'search_code',
+    glob: 'list_files',
+    list: 'list_files',
+    skill: 'use_skill',
+    todowrite: 'update_todos',
+    lsp: 'search_code',
+};
+export const ALLOWED_EVENT_TYPES = new Set([
+    'tool_use',
+    'step_start',
+    'step_finish',
+    'text',
+    'reasoning',
+    'error',
+]);
+export function protocolRecord(value, label) {
+    if (!value || typeof value !== 'object' || Array.isArray(value)) {
+        throw new Error(`OpenCode protocol error: ${label} must be an object`);
+    }
+    return value;
+}
+export function protocolFiniteNumber(value, label) {
+    if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
+        throw new Error(`OpenCode protocol error: ${label} must be a nonnegative finite number`);
+    }
+    return value;
+}
+export function protocolRequiredString(value, label) {
+    if (typeof value !== 'string' || !value) {
+        throw new Error(`OpenCode protocol error: ${label} must be a nonempty string`);
+    }
+    return value;
+}
+export function sanitizedProviderError(event) {
+    const error = protocolRecord(event.error, 'error');
+    const data = error.data && typeof error.data === 'object' && !Array.isArray(error.data)
+        ? error.data
+        : {};
+    const status = typeof data.statusCode === 'number' ? ` status=${data.statusCode}` : '';
+    const retryable = typeof data.isRetryable === 'boolean' ? ` retryable=${data.isRetryable}` : '';
+    return new Error(`OpenCode provider error${status}${retryable}`);
+}

package/dist/agents/opencode.js

@@ -6,7 +6,7 @@ import { buildSummary, enrichSkillEvents } from '../tool-events.js';
 import { collectSensitiveEnvValues, sanitizePersistenceValue, sanitizeToolEventResult, } from '../tool-event-results.js';
 import { readStagedMcpServers } from '../providers/mcp-config.js';
 import { removeSandboxRoot } from '../providers/sandbox-lifecycle.js';
-import { attachTurnResultSensitiveValues } from '../sdk/turn-result-secrets.js';
+import { attachTurnResultSensitiveValues, cloneTurnResultWithSensitiveValues } from '../sdk/turn-result-secrets.js';
 import { attachOriginalMcpInput } from '../sdk/mcp-event-input.js';
 import { attachToolEventSensitiveValues } from '../sdk/tool-event-secrets.js';
 import { currentOpenCodePlatformKey, OPENCODE_RUNTIME_LOCK, } from './opencode/contract.js';
@@ -14,28 +14,9 @@ import { OpenCodeRuntimePolicy, OPENCODE_PERMISSION } from './opencode/runtime-p
 import { killOpenCodeProcessGroup, registerOpenCodeProcessGroup, unregisterOpenCodeProcessGroup, } from './opencode/process-groups.js';
 import { assertCleanManagedOpenCodeHost, sha256File } from './opencode/host-safety.js';
 import { getOpenCodeScenarioSession } from './opencode/scenario.js';
+import { ALLOWED_EVENT_TYPES, NATIVE_TOOL_ACTIONS, protocolFiniteNumber as finiteNumber, protocolRecord as record, protocolRequiredString as requiredString, sanitizedProviderError, } from './opencode/protocol.js';
 export { assertCleanManagedOpenCodeHost, managedOpenCodeConfigPaths } from './opencode/host-safety.js';
 const OUTPUT_CAP_BYTES = 16 * 1024 * 1024;
-const NATIVE_TOOL_ACTIONS = {
-    bash: 'run_shell',
-    read: 'read_file',
-    write: 'write_file',
-    edit: 'edit_file',
-    grep: 'search_code',
-    glob: 'list_files',
-    list: 'list_files',
-    skill: 'use_skill',
-    todowrite: 'update_todos',
-    lsp: 'search_code',
-};
-const ALLOWED_EVENT_TYPES = new Set([
-    'tool_use',
-    'step_start',
-    'step_finish',
-    'text',
-    'reasoning',
-    'error',
-]);
 export function spawnOpenCode(executable, args, options) {
     return new Promise((resolve, reject) => {
         const stdout = [];
@@ -124,33 +105,6 @@ export function spawnOpenCode(executable, args, options) {
         }
     });
 }
-function record(value, label) {
-    if (!value || typeof value !== 'object' || Array.isArray(value)) {
-        throw new Error(`OpenCode protocol error: ${label} must be an object`);
-    }
-    return value;
-}
-function finiteNumber(value, label) {
-    if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {
-        throw new Error(`OpenCode protocol error: ${label} must be a nonnegative finite number`);
-    }
-    return value;
-}
-function requiredString(value, label) {
-    if (typeof value !== 'string' || !value) {
-        throw new Error(`OpenCode protocol error: ${label} must be a nonempty string`);
-    }
-    return value;
-}
-function sanitizedProviderError(event) {
-    const error = record(event.error, 'error');
-    const data = error.data && typeof error.data === 'object' && !Array.isArray(error.data)
-        ? error.data
-        : {};
-    const status = typeof data.statusCode === 'number' ? ` status=${data.statusCode}` : '';
-    const retryable = typeof data.isRetryable === 'boolean' ? ` retryable=${data.isRetryable}` : '';
-    return new Error(`OpenCode provider error${status}${retryable}`);
-}
 export function parseOpenCodeOutput(stdout, processResult, mcpToolNames, sensitiveValues = []) {
     if (processResult.overflow)
         throw new Error('OpenCode output exceeded the 16 MiB limit');
@@ -451,7 +405,9 @@ class OpenCodeSession {
                 throw new Error('OpenCode protocol error: resumed session ID changed');
             }
             this.sessionId = parsed.sessionId;
-            return parsed.result;
+            return cloneTurnResultWithSensitiveValues(parsed.result, {
+                resolvedModel: this.runtimePolicy.model,
+            });
         }
         catch (error) {
             this.failed = true;

package/dist/analytics/engine.js

@@ -40,10 +40,10 @@ export class AnalyticsEngine {
         for (const [task, data] of Object.entries(taskGroups)) {
             const allReports = [...data.withSkill, ...data.withoutSkill];
             const avgWith = data.withSkill.length > 0
-                ? data.withSkill.reduce((a, b) => a + b.pass_rate, 0) / data.withSkill.length
+                ? data.withSkill.reduce((a, b) => a + reportMeanReward(b), 0) / data.withSkill.length
                 : 0;
             const avgWithout = data.withoutSkill.length > 0
-                ? data.withoutSkill.reduce((a, b) => a + b.pass_rate, 0) / data.withoutSkill.length
+                ? data.withoutSkill.reduce((a, b) => a + reportMeanReward(b), 0) / data.withoutSkill.length
                 : 0;
             const allTrials = allReports.flatMap(r => r.trials);
             const avgDurationMs = allTrials.length > 0
@@ -64,3 +64,6 @@ export class AnalyticsEngine {
         return stats;
     }
 }
+function reportMeanReward(report) {
+    return report.mean_reward ?? report.pass_rate ?? 0;
+}

package/dist/commands/report.d.ts

@@ -24,6 +24,14 @@ export interface ReportOptions {
     noComment?: boolean;
     /** Override the comment-id used in the dedup marker. See `resolveCommentId`. */
     commentId?: string;
+    /** Provider-neutral URL for a richer external report. */
+    detailsUrl?: string;
+    /** Provider-neutral status notice included in the report body. */
+    notice?: string;
+    /** Surface missing context, malformed results, and GitHub API failures. */
+    strict?: boolean;
+    /** Skip a comment update when the pull request has advanced. */
+    expectedHeadSha?: string;
 }
 /**
  * Resolve the comment-id used for the dedup marker.
@@ -33,7 +41,7 @@ export interface ReportOptions {
  */
 export declare function resolveCommentId(explicit?: string): string;
 /**
- * Entry point for `pathgrade report`. Never throws and never sets a
- * failing exit code — even on malformed inputs or network errors.
+ * Entry point for `pathgrade report`. The default display-only mode never
+ * fails CI. Strict mode is intended for orchestrators that own publication.
  */
 export declare function runReport(cwd: string, opts?: ReportOptions): Promise<void>;

package/dist/commands/report.js

@@ -43,10 +43,24 @@ function isPathgradeReport(value) {
     if (!value || typeof value !== 'object')
         return false;
     const v = value;
-    return (v.version === 1 &&
+    return ((v.version === 1 || v.version === 2) &&
         typeof v.overall_pass_rate === 'number' &&
         (v.status === 'pass' || v.status === 'fail') &&
-        Array.isArray(v.groups));
+        Array.isArray(v.groups) &&
+        (v.version === 1 || (typeof v.overall_mean_reward === 'number'
+            && hasValidAttemptCounts(v)
+            && v.groups.every(group => typeof group.mean_reward === 'number'))));
+}
+function hasValidAttemptCounts(report) {
+    const requested = report.attempts_requested;
+    const completed = report.attempts_completed;
+    return typeof requested === 'number'
+        && Number.isSafeInteger(requested)
+        && requested >= 1
+        && typeof completed === 'number'
+        && Number.isSafeInteger(completed)
+        && completed >= 0
+        && completed <= requested;
 }
 async function loadReport(resolvedPath) {
     if (!(await fs.pathExists(resolvedPath))) {
@@ -66,8 +80,8 @@ function printMarkdownAndPassRate(markdown, passRate) {
     console.log(String(passRate));
 }
 /**
- * Entry point for `pathgrade report`. Never throws and never sets a
- * failing exit code — even on malformed inputs or network errors.
+ * Entry point for `pathgrade report`. The default display-only mode never
+ * fails CI. Strict mode is intended for orchestrators that own publication.
  */
 export async function runReport(cwd, opts = {}) {
     const resolvedPath = path.resolve(cwd, opts.resultsPath ?? DEFAULT_RESULTS_PATH);
@@ -77,6 +91,9 @@ export async function runReport(cwd, opts = {}) {
     // Branch point: when true, post the comment and print only the pass rate.
     const shouldPost = inCI && !noComment;
     const prContext = shouldPost ? resolvePrContext(process.env) : null;
+    if (opts.strict && shouldPost && !prContext) {
+        throw new Error('pathgrade report: strict PR commenting requires GITHUB_TOKEN, GITHUB_REPOSITORY, and pull-request context');
+    }
     let report = null;
     let loadError = null;
     try {
@@ -95,6 +112,8 @@ export async function runReport(cwd, opts = {}) {
                 await postOrUpdateComment(prContext, {
                     commentId,
                     body: markdown,
+                    strict: opts.strict,
+                    expectedHeadSha: opts.expectedHeadSha,
                 });
                 console.log('0');
             }
@@ -108,15 +127,31 @@ export async function runReport(cwd, opts = {}) {
             await postOrUpdateComment(prContext, {
                 commentId,
                 body: MISSING_RESULTS_BODY,
+                strict: opts.strict,
+                expectedHeadSha: opts.expectedHeadSha,
             });
         }
         // Always emit a pass rate line so downstream capture doesn't explode.
         console.log('0');
+        if (opts.strict)
+            throw new Error(`pathgrade report: ${loadError}`);
         return;
     }
-    const markdown = formatReportMarkdown(report, { commentId });
+    const markdown = formatReportMarkdown(report, {
+        commentId,
+        detailsUrl: opts.detailsUrl,
+        notice: opts.notice,
+    });
     if (prContext) {
-        await postOrUpdateComment(prContext, { commentId, body: markdown });
+        const postResult = await postOrUpdateComment(prContext, {
+            commentId,
+            body: markdown,
+            strict: opts.strict,
+            expectedHeadSha: opts.expectedHeadSha,
+        });
+        if (postResult === 'stale') {
+            console.error('pathgrade report: pull-request head advanced; skipped stale comment update');
+        }
         // Only the pass rate goes to stdout — the markdown lives on the PR.
         console.log(String(report.overall_pass_rate));
         return;

package/dist/commands/run-args.d.ts

@@ -8,6 +8,7 @@
 export interface PathgradeRunArgs {
     runnerArgs: string[];
     adapterName?: string;
+    attempts?: number;
     forceDiagnostics: boolean;
     forceVerbose: boolean;
     changed: boolean;

package/dist/commands/run-args.js

@@ -14,6 +14,7 @@ export function parsePathgradeRunArgs(args) {
     let since;
     let changedFilesPath;
     let adapterName;
+    let attempts;
     let passthrough = false;
     for (const arg of args) {
         if (passthrough) {
@@ -44,6 +45,10 @@ export function parsePathgradeRunArgs(args) {
             adapterName = arg.slice('--adapter='.length);
             continue;
         }
+        if (arg.startsWith('--attempts=')) {
+            attempts = parseAttempts(arg.slice('--attempts='.length));
+            continue;
+        }
         if (arg.startsWith('--since=')) {
             since = arg.slice('--since='.length);
             continue;
@@ -68,8 +73,16 @@ export function parsePathgradeRunArgs(args) {
         changed,
         quiet,
         adapterName,
+        attempts,
         since,
         changedFilesPath,
     };
     return warnings.length > 0 ? { ...base, warnings } : base;
 }
+function parseAttempts(value) {
+    const attempts = Number(value);
+    if (!Number.isSafeInteger(attempts) || attempts < 1) {
+        throw new Error('pathgrade: --attempts must be an integer greater than or equal to 1');
+    }
+    return attempts;
+}

package/dist/commands/run-changed.js

@@ -27,13 +27,13 @@ export async function runChanged(opts) {
     const runnerEnv = buildRunnerEnv(parsed, {
         PATHGRADE_SELECTION_INVOCATION_ID: selectionInvocationId,
     });
-    const configPath = findVitestConfigArg(parsed.runnerArgs);
     let config;
     let runnerInvocation;
     try {
         config = await resolvePathgradeConfig({
             cwd,
-            legacyVitestConfigPath: configPath,
+            cli: parsed.attempts === undefined ? undefined : { attempts: parsed.attempts },
+            runnerArgs: parsed.runnerArgs,
             warn: w => {
                 if (!parsed.quiet)
                     process.stderr.write(`${w}\n`);
@@ -107,6 +107,7 @@ export async function runChanged(opts) {
             totalEvals: evalFiles.length,
             changedCount: changedFiles.length,
             result,
+            attempts: config.attempts,
         });
     }
     // Persist the sidecar immediately — even on empty selection, so the
@@ -128,7 +129,7 @@ export async function runChanged(opts) {
         cwd,
         runnerArgs,
         selectedFiles,
-        env: runnerEnv,
+        env: { ...runnerEnv, PATHGRADE_ATTEMPT_COUNT: String(config.attempts) },
     });
 }
 function printRunStartSummary(input) {
@@ -140,6 +141,7 @@ function printRunStartSummary(input) {
     const globalLabel = result.globalMatch ? `\`${result.globalMatch}\`` : 'none';
     process.stderr.write(`          global matches: ${globalLabel}\n`);
     process.stderr.write(`          selected: ${result.selected.length} / ${totalEvals} evals\n`);
+    process.stderr.write(`          attempts: ${input.attempts} per selected case\n`);
     for (const entry of result.selected) {
         process.stderr.write(`            ${entry.file}\n`);
     }
@@ -158,15 +160,3 @@ function readChangedFilesList(filePath) {
 function errMsg(err) {
     return err instanceof Error ? err.message : String(err);
 }
-function findVitestConfigArg(args) {
-    for (let i = 0; i < args.length; i++) {
-        const arg = args[i];
-        if (arg === '--config' || arg === '-c')
-            return args[i + 1];
-        if (arg.startsWith('--config='))
-            return arg.slice('--config='.length);
-        if (arg.startsWith('-c='))
-            return arg.slice('-c='.length);
-    }
-    return undefined;
-}

package/dist/config/pathgrade.d.ts

@@ -1,4 +1,5 @@
 export interface PathgradeConfig {
+    attempts?: number;
     runner?: {
         adapter?: string;
         args?: string[];
@@ -18,6 +19,7 @@ export interface PathgradeConfig {
     };
 }
 export interface ResolvedPathgradeConfig {
+    attempts: number;
     runner: {
         adapter: string;
         args: string[];
@@ -44,5 +46,6 @@ export declare function resolvePathgradeConfig(input: {
     cli?: PathgradeConfig;
     configPath?: string;
     legacyVitestConfigPath?: string;
+    runnerArgs?: readonly string[];
     warn?: (message: string) => void;
 }): Promise<ResolvedPathgradeConfig>;

package/dist/config/pathgrade.js

@@ -17,6 +17,7 @@ const PATHGRADE_CONFIG_CANDIDATES = [
 ];
 export function defaultPathgradeConfig() {
     return {
+        attempts: 1,
         runner: {
             adapter: 'vitest',
             args: [],
@@ -35,13 +36,30 @@ export function defaultPathgradeConfig() {
 }
 export async function resolvePathgradeConfig(input) {
     const fileConfig = await loadPathgradeConfigFile(input.cwd, input.configPath);
-    const legacyConfig = await loadLegacyVitestConfig(input.cwd, input.legacyVitestConfigPath, input.warn);
+    const legacyConfig = await loadLegacyVitestConfig(input.cwd, input.legacyVitestConfigPath ?? findVitestConfigArg([
+        ...(fileConfig?.runner?.args ?? []),
+        ...(input.runnerArgs ?? []),
+    ]), input.warn);
     return mergePathgradeConfig(mergePathgradeConfig(mergePathgradeConfig(defaultPathgradeConfig(), legacyConfig), fileConfig), input.cli);
 }
+function findVitestConfigArg(args) {
+    let configPath;
+    for (let i = 0; i < args.length; i++) {
+        const arg = args[i];
+        if (arg === '--config' || arg === '-c')
+            configPath = args[i + 1];
+        else if (arg.startsWith('--config='))
+            configPath = arg.slice('--config='.length);
+        else if (arg.startsWith('-c='))
+            configPath = arg.slice('-c='.length);
+    }
+    return configPath;
+}
 function mergePathgradeConfig(base, override) {
     if (!override)
         return base;
     return {
+        attempts: override.attempts ?? base.attempts,
         runner: {
             adapter: override.runner?.adapter ?? base.runner.adapter,
             args: override.runner?.args ?? base.runner.args,
@@ -91,6 +109,7 @@ function validatePathgradeConfig(value, label) {
         throw invalidConfig(label, 'default export must be an object');
     }
     validateOptionalObject(value.runner, label, 'runner');
+    validateOptionalPositiveInteger(value.attempts, label, 'attempts');
     const runner = asOptionalObject(value.runner);
     validateOptionalString(runner?.adapter, label, 'runner.adapter');
     validateOptionalStringArray(runner?.args, label, 'runner.args');
@@ -139,6 +158,11 @@ function validateOptionalNumber(value, label, field) {
         throw invalidConfig(label, `${field} must be a number`);
     }
 }
+function validateOptionalPositiveInteger(value, label, field) {
+    if (value !== undefined && (!Number.isSafeInteger(value) || Number(value) < 1)) {
+        throw invalidConfig(label, `${field} must be an integer greater than or equal to 1`);
+    }
+}
 function invalidConfig(label, reason) {
     return new InvalidPathgradeConfigError(`pathgrade: invalid ${label}: ${reason}`);
 }
@@ -179,6 +203,14 @@ async function loadLegacyVitestConfig(cwd, configPath, warn = () => { }) {
     if (!isObject(opts))
         return undefined;
     return {
+        ...(opts.reporter === 'cli' || opts.reporter === 'browser' || opts.reporter === 'json'
+            ? { reporter: opts.reporter }
+            : {}),
+        ...(typeof opts.diagnostics === 'boolean' ? { diagnostics: opts.diagnostics } : {}),
+        ...(typeof opts.verbose === 'boolean' ? { verbose: opts.verbose } : {}),
+        ...(isObject(opts.ci) && typeof opts.ci.threshold === 'number'
+            ? { ci: { threshold: opts.ci.threshold } }
+            : {}),
         evals: {
             ...(Array.isArray(opts.include) ? { include: opts.include } : {}),
             ...(Array.isArray(opts.exclude) ? { exclude: opts.exclude } : {}),

package/dist/pathgrade.js

@@ -156,10 +156,27 @@ export async function runPathgradeCli(options = {}) {
         const reportArgs = args.slice(1);
         const resultsPathFlag = reportArgs.find(a => a.startsWith('--results-path='));
         const commentIdFlag = reportArgs.find(a => a.startsWith('--comment-id='));
+        const detailsUrlFlag = reportArgs.find(a => a.startsWith('--details-url='));
+        const noticeFlag = reportArgs.find(a => a.startsWith('--notice='));
+        const expectedHeadShaFlag = reportArgs.find(a => a.startsWith('--expected-head-sha='));
         const noComment = reportArgs.includes('--no-comment');
+        const strict = reportArgs.includes('--strict');
         const resultsPath = resultsPathFlag ? resultsPathFlag.split('=').slice(1).join('=') : undefined;
         const commentId = commentIdFlag ? commentIdFlag.split('=').slice(1).join('=') : undefined;
-        await runReport(process.cwd(), { resultsPath, commentId, noComment });
+        const detailsUrl = detailsUrlFlag ? detailsUrlFlag.split('=').slice(1).join('=') : undefined;
+        const notice = noticeFlag ? noticeFlag.split('=').slice(1).join('=') : undefined;
+        const expectedHeadSha = expectedHeadShaFlag
+            ? expectedHeadShaFlag.split('=').slice(1).join('=')
+            : undefined;
+        await runReport(process.cwd(), {
+            resultsPath,
+            commentId,
+            detailsUrl,
+            notice,
+            expectedHeadSha,
+            noComment,
+            strict,
+        });
         return;
     }
     if (command === 'run' || !command || command.startsWith('-')) {
@@ -184,7 +201,14 @@ export async function runPathgradeCli(options = {}) {
         await clearSidecar(process.cwd());
         const env = buildRunnerEnv(parsed);
         try {
-            const config = await resolvePathgradeConfig({ cwd: process.cwd() });
+            const config = await resolvePathgradeConfig({
+                cwd: process.cwd(),
+                cli: parsed.attempts === undefined ? undefined : { attempts: parsed.attempts },
+                runnerArgs: parsed.runnerArgs,
+            });
+            if (!parsed.quiet && config.attempts > 1) {
+                console.error(`pathgrade: ${config.attempts} sequential attempts per selected case`);
+            }
             const runner = await loadRunnerInvocationAdapter({
                 adapterName: parsed.adapterName ?? config.runner.adapter,
                 cwd: process.cwd(),
@@ -193,7 +217,7 @@ export async function runPathgradeCli(options = {}) {
             process.exitCode = await runner.run({
                 cwd: process.cwd(),
                 runnerArgs: [...config.runner.args, ...parsed.runnerArgs],
-                env,
+                env: { ...env, PATHGRADE_ATTEMPT_COUNT: String(config.attempts) },
             });
         }
         catch (err) {
@@ -216,6 +240,7 @@ function printHelp(cliName) {
                      [--since=<ref>]           Override base ref (implies git mode)
                      [--changed-files=<path>]  Use an explicit newline-delimited file list
                      [--adapter=<name|path>]   Select built-in or third-party runner adapter
+                     [--attempts=N]            Run every selected case N times sequentially
                      [--diagnostics]           Print full diagnostics for passing evals too
                      [--quiet]                 Suppress the run-start summary
                      [--verbose|-v]            Stream live per-turn events to stderr during the run
@@ -234,6 +259,10 @@ function printHelp(cliName) {
                      [--results-path=<path>]   Override results.json location
                      [--no-comment]            Print markdown to stdout; do not post
                      [--comment-id=<id>]       Override comment marker (default: $GITHUB_WORKFLOW:$GITHUB_JOB)
+                     [--details-url=<url>]     Add a provider-neutral full-report link
+                     [--notice=<text>]         Add a provider-neutral report status notice
+                     [--expected-head-sha=<sha>] Skip stale PR comment updates
+                     [--strict]                Fail on missing context, results, or GitHub errors
     ${cliName} affected               Print eval files affected by a change-set (one per line)
                      [--since=<ref>]           Diff <ref>...HEAD (overrides git auto-detection)
                      [--changed-files=<path>]  Newline-delimited repo-relative file list

package/dist/reporters/cli.js

@@ -18,8 +18,8 @@ export async function runCliPreview(resultsDir, opts) {
     }
     console.log(`\n${fmt.bold('pathgrade preview')}  ${fmt.dim(`${entries.length} reports from ${resolved}`)}\n`);
     for (const { file, ...report } of entries) {
-        const passRate = report.pass_rate ?? 0;
-        const isPass = passRate >= 0.5;
+        const meanReward = report.mean_reward ?? report.pass_rate ?? 0;
+        const isPass = report.status === undefined ? meanReward >= 0.5 : report.status === 'pass';
         const trials = report.trials || [];
         const avgDur = trials.reduce((s, t) => s + (t.duration_ms || 0), 0) / (trials.length || 1);
         const totalTokens = trials.reduce((s, t) => s + (t.input_tokens || 0) + (t.output_tokens || 0) + (t.conversation_input_tokens || 0) + (t.conversation_output_tokens || 0), 0);
@@ -33,9 +33,9 @@ export async function runCliPreview(resultsDir, opts) {
         console.log();
         // ── Summary metrics
         const metrics = [
-            ['Pass Rate', `${(passRate * 100).toFixed(1)}%`],
-            ['pass@k', report.pass_at_k != null ? `${(report.pass_at_k * 100).toFixed(1)}%` : '—'],
-            ['pass^k', report.pass_pow_k != null ? `${(report.pass_pow_k * 100).toFixed(1)}%` : '—'],
+            ['Mean Reward', `${(meanReward * 100).toFixed(1)}%`],
+            ['Success Rate', report.success_rate != null ? `${(report.success_rate * 100).toFixed(1)}%` : '—'],
+            ['pass@k', formatPassAtK(report.pass_at_k, report.pass_at_k_unavailable_reason)],
             ['Avg Duration', `${(avgDur / 1000).toFixed(1)}s`],
             ['Total Tokens', `~${totalTokens}`],
             ['Skills', report.skills_used?.join(', ') || 'none'],
@@ -47,7 +47,7 @@ export async function runCliPreview(resultsDir, opts) {
         // ── Trials
         for (const trial of trials) {
             const evaluated = trial.reward !== undefined;
-            const tp = evaluated && trial.reward >= 0.5;
+            const tp = evaluated && (trial.runner_outcome === undefined ? trial.reward >= 0.5 : trial.runner_outcome === 'passed' && trial.reward === 1);
             const trialStatus = !evaluated ? fmt.dim('N/A') : tp ? fmt.pass('PASS') : fmt.fail('FAIL');
             const reward = fmt.bold(evaluated ? trial.reward.toFixed(2) : 'n/a');
             const dur = `${((trial.duration_ms || 0) / 1000).toFixed(1)}s`;
@@ -120,6 +120,13 @@ export async function runCliPreview(resultsDir, opts) {
         console.log();
     }
 }
+function formatPassAtK(value, reason) {
+    if (typeof value === 'number')
+        return `${(value * 100).toFixed(1)}% (legacy v1)`;
+    if (value)
+        return Object.entries(value).map(([k, metric]) => `@${k} ${(metric * 100).toFixed(1)}%`).join(', ');
+    return reason ? `— (${reason.replaceAll('_', ' ')})` : '—';
+}
 function formatScorerStatus(status) {
     switch (status) {
         case 'error':

package/dist/reporters/github-comment.d.ts

@@ -10,6 +10,10 @@
 import type { PathgradeReport, PathgradeSelectionReport } from '../types.js';
 export interface FormatOptions {
     commentId: string;
+    /** Provider-neutral URL for a richer external report. */
+    detailsUrl?: string;
+    /** Provider-neutral status notice rendered above report details. */
+    notice?: string;
 }
 /** Minimal body posted when `.pathgrade/results.json` is missing. */
 export declare const MISSING_RESULTS_BODY = "Pathgrade evals did not produce results. Check the workflow logs.";
@@ -55,14 +59,19 @@ export interface PostOptions {
     commentId: string;
     /** Body WITHOUT the marker — the marker is prepended if not already present. */
     body: string;
+    /** Throw instead of swallowing GitHub API failures. */
+    strict?: boolean;
+    /** Skip the update if the pull request has advanced to another head. */
+    expectedHeadSha?: string;
 }
+export type PostResult = 'created' | 'updated' | 'stale';
 /**
  * Find an existing PR comment carrying `<!-- pathgrade:${commentId} -->`
- * and update it; otherwise create a new one. Swallows all errors (logs
- * to stderr) — `pathgrade report` must never fail CI.
+ * and update it; otherwise create a new one. Default mode logs and swallows
+ * transport errors; strict mode rethrows them for an owning orchestrator.
  *
  * Bodies longer than GitHub's 65,536-character limit are truncated with a
  * sentinel pointing to workflow artifacts, preserving the leading dedup
  * marker so subsequent runs still find and update this comment.
  */
-export declare function postOrUpdateComment(ctx: PrContext, opts: PostOptions): Promise<void>;
+export declare function postOrUpdateComment(ctx: PrContext, opts: PostOptions): Promise<PostResult | undefined>;