@wix/pathgrade 1.0.38 → 1.0.39

package/README.md

@@ -461,7 +461,7 @@ export default {
 
 Pathgrade reads `pathgrade.config.*` for CLI and affected-selection behavior. `runner.adapter` and `--adapter=<name|path>` select the runner; `--adapter` wins over config. `attempts` defaults to `1` and must be a positive integer. Built-in adapters `vitest`, `jest`, and `node-test` support repeated attempts; third-party invocation adapters must advertise `supportsRepeatedAttempts: true` and produce the normalized child snapshot contract.
 
-New reports use schema version 2. They preserve every attempt with `case_id`, `attempt_id`, and `attempt_index`; include the display-safe `runner_outcome` that determined binary success; publish `mean_reward` for partial scores; and publish finite-sample pass@k only for complete binary attempts. Runner assertions, runner diagnostics, and native runner references remain in the normalized run model and are not copied into public report artifacts; evaluation diagnostics retain their existing report behavior. A partial reward or incomplete attempt makes pass@k explicitly unavailable. Version-1 reports remain readable, but Pathgrade no longer computes pass@k or pass^k by pooling heterogeneous cases.
+New reports use schema version 2. They preserve every attempt with `case_id`, `attempt_id`, and `attempt_index`; include the display-safe `runner_outcome` that determined binary success; publish `mean_reward` for partial scores; and publish finite-sample pass@k only for complete binary attempts. `runner_status` and `threshold_status` expose the two gates independently, and canonical `status` passes only when runner assertions pass and any configured threshold passes. Runner assertions, runner diagnostics, and native runner references remain in the normalized run model and are not copied into public report artifacts; evaluation diagnostics retain their existing report behavior. A partial reward or incomplete attempt makes pass@k explicitly unavailable. Version-1 and older version-2 reports remain readable with missing gate fields derived during loading, but Pathgrade no longer computes pass@k or pass^k by pooling heterogeneous cases.
 
 Third-party runner adapters are supported through `@wix/pathgrade/adapter-kit`. Adapter names resolve as follows:
 

package/dist/agents/claude/sdk-message-projector.js

@@ -22,10 +22,9 @@ const SDK_ERROR_SUBTYPES = [
     'error_max_structured_output_retries',
 ];
 import { TOOL_NAME_MAP, buildSummary, enrichSkillEvents } from '../../tool-events.js';
-import { sanitizePersistenceValue } from '../../tool-event-results.js';
+import { sanitizePersistenceValue, sanitizeUntrustedPersistenceValue } from '../../tool-event-results.js';
 import { attachTurnResultSensitiveValues } from '../../sdk/turn-result-secrets.js';
-import { attachToolEventSensitiveValues } from '../../sdk/tool-event-secrets.js';
-import { attachOriginalMcpInput } from '../../sdk/mcp-event-input.js';
+import { attachLiveMcpInput, attachToolEventSensitiveValues } from '../../sdk/tool-event-secrets.js';
 import { parseClaudeSdkMcpToolName } from './mcp-tool-name.js';
 import { applyObservedToolResult, extractObservedToolResults, } from './tool-results.js';
 export function projectSdkMessages(input) {
@@ -145,7 +144,7 @@ export function projectSdkMessages(input) {
         ...(input.deniedMcpEvents?.all() ?? []),
     ]);
     const finalToolEvents = prependSlashCommandSkillEvent(enriched, input.firstMessage, initSkills).map((event) => attachToolEventSensitiveValues(event, input.sensitiveValues ?? []));
-    const traceOutput = sanitizePersistenceValue(input.messages, input.sensitiveValues)
+    const traceOutput = sanitizeUntrustedPersistenceValue(input.messages, input.sensitiveValues)
         .map((message) => JSON.stringify(message))
         .join('\n');
     const result = {
@@ -216,7 +215,7 @@ function buildToolEvent(block, turnNumber, answerStore, mcpServerNames, deniedMc
             tool: mcpTool.tool,
             status: 'incomplete',
         };
-        return attachOriginalMcpInput({
+        return attachLiveMcpInput({
             action: 'mcp_tool_call',
             provider: 'claude',
             providerToolName: normalizedProviderToolName,

package/dist/agents/claude/tool-permission-bridge.js

@@ -3,6 +3,7 @@ import { createAskUserBridge, } from './ask-user-bridge.js';
 import { parseClaudeSdkMcpToolName } from './mcp-tool-name.js';
 import { canonicalizeJson } from '../../core/canonical-json.js';
 import { buildScriptedMcpApprovalEvent, buildScriptedMcpDeniedCallEvent, } from '../../sdk/scripted-mcp-events.js';
+import { attachLiveMcpInput } from '../../sdk/tool-event-secrets.js';
 export function createClaudeToolPermissionBridge(deps) {
     const askUserBridge = createAskUserBridge({
         askBus: deps.askBus,
@@ -149,5 +150,5 @@ function recordDeniedMcpEvent(opts) {
             input: opts.input,
         })).slice(0, 200),
     };
-    opts.store.record(opts.toolUseId, event);
+    opts.store.record(opts.toolUseId, attachLiveMcpInput(event, opts.input));
 }

package/dist/agents/codex-app-server/item-projection.js

@@ -1,6 +1,6 @@
 import { buildSummary, extractSkillNameFromPath, inferCodexExecAction, } from '../../tool-events.js';
 import { sanitizeToolEventResult } from '../../tool-event-results.js';
-import { attachOriginalMcpInput } from '../../sdk/mcp-event-input.js';
+import { attachLiveMcpInput } from '../../sdk/tool-event-secrets.js';
 export function projectItemIntoTurn(item, turn, sensitiveValues, timing = {}) {
     if (item.type === 'agentMessage') {
         const message = item;
@@ -76,7 +76,7 @@ function projectMcpCall(call, turn, sensitiveValues, timing) {
     const status = call.status === 'completed' && error === undefined
         ? 'completed' : call.status === 'failed' || error !== undefined ? 'error' : 'incomplete';
     const resultContent = call.result === undefined ? undefined : JSON.stringify(call.result);
-    turn.nonAskToolEvents.push(attachOriginalMcpInput({
+    turn.nonAskToolEvents.push(attachLiveMcpInput({
         action: 'mcp_tool_call', provider: 'codex', providerToolName, toolUseId: call.id,
         turnNumber: turn.turnNumber, status, ...projectToolTiming(timing, finiteNumber(call.durationMs)),
         arguments: { ...args, server: call.server, tool: call.tool, status: call.status ?? 'unknown' },

package/dist/agents/codex-app-server/mcp-approval-correlator.js

@@ -1,6 +1,7 @@
 import { decideMcpToolCall, redactMcpSecrets, } from '../../sdk/mcp-safety.js';
 import { canonicalizeJson } from '../../core/canonical-json.js';
 import { buildScriptedMcpApprovalEvent, buildScriptedMcpDeniedCallEvent, } from '../../sdk/scripted-mcp-events.js';
+import { attachLiveMcpInput } from '../../sdk/tool-event-secrets.js';
 function isRecord(value) {
     return !!value && typeof value === 'object' && !Array.isArray(value);
 }
@@ -64,7 +65,7 @@ export function queuePolicyDeniedMcpToolCall(turn, request, decision, rawParams)
 export function buildPolicyDeniedMcpToolEvent(turnNumber, pending, terminal) {
     const args = redactMcpSecrets(pending.request.arguments);
     const providerToolName = `${pending.request.serverName}.${pending.request.toolName}`;
-    return {
+    return attachLiveMcpInput({
         action: 'mcp_tool_call', provider: 'codex', providerToolName,
         ...(terminal ? { toolUseId: terminal.id } : {}),
         turnNumber, status: 'error',
@@ -79,7 +80,7 @@ export function buildPolicyDeniedMcpToolEvent(turnNumber, pending, terminal) {
         },
         summary: `MCP tool ${providerToolName} policy_denied`, confidence: 'high',
         rawSnippet: JSON.stringify(redactMcpSecrets(terminal ? { approval: pending.rawParams, terminal } : pending.rawParams)),
-    };
+    }, pending.request.arguments);
 }
 export function consumeMatchingMcpDenial(turn, serverName, toolName, args) {
     const argsKey = canonicalizeJson(args);

package/dist/agents/opencode.js

@@ -3,12 +3,11 @@ import * as path from 'node:path';
 import fs from 'fs-extra';
 import { BaseAgent, getRuntimeEnv, getWorkspacePath, } from '../types.js';
 import { buildSummary, enrichSkillEvents } from '../tool-events.js';
-import { collectSensitiveEnvValues, sanitizePersistenceValue, sanitizeToolEventResult, } from '../tool-event-results.js';
+import { collectSensitiveEnvValues, sanitizeToolEventResult, sanitizeUntrustedPersistenceValue, } from '../tool-event-results.js';
 import { readStagedMcpServers } from '../providers/mcp-config.js';
 import { removeSandboxRoot } from '../providers/sandbox-lifecycle.js';
 import { attachTurnResultSensitiveValues, cloneTurnResultWithSensitiveValues } from '../sdk/turn-result-secrets.js';
-import { attachOriginalMcpInput } from '../sdk/mcp-event-input.js';
-import { attachToolEventSensitiveValues } from '../sdk/tool-event-secrets.js';
+import { attachLiveMcpInput, attachToolEventSensitiveValues } from '../sdk/tool-event-secrets.js';
 import { currentOpenCodePlatformKey, OPENCODE_RUNTIME_LOCK, } from './opencode/contract.js';
 import { OpenCodeRuntimePolicy, OPENCODE_PERMISSION } from './opencode/runtime-policy.js';
 import { killOpenCodeProcessGroup, registerOpenCodeProcessGroup, unregisterOpenCodeProcessGroup, } from './opencode/process-groups.js';
@@ -217,7 +216,7 @@ export function parseOpenCodeOutput(stdout, processResult, mcpToolNames, sensiti
                 rawSnippet: JSON.stringify({ tool, status: state.status, input }).slice(0, 2_000),
             };
             toolEvents.push(action === 'mcp_tool_call' && input
-                ? attachOriginalMcpInput(toolEvent, input)
+                ? attachLiveMcpInput(toolEvent, input)
                 : toolEvent);
             sanitizedTrace.push({ type, tool, status, input });
             continue;
@@ -256,7 +255,7 @@ export function parseOpenCodeOutput(stdout, processResult, mcpToolNames, sensiti
     if (stepFinishCount === 0)
         throw new Error('OpenCode protocol error: missing step_finish');
     const assistantMessage = textParts.join('');
-    const traceOutput = sanitizePersistenceValue(sanitizedTrace, sensitiveValues)
+    const traceOutput = sanitizeUntrustedPersistenceValue(sanitizedTrace, sensitiveValues)
         .map((event) => JSON.stringify(event))
         .join('\n');
     return {

package/dist/commands/report.js

@@ -20,6 +20,7 @@
 import * as path from 'path';
 import fs from 'fs-extra';
 import { readSidecar } from '../affected/sidecar.js';
+import { parsePathgradeReport } from '../reporting/report-parser.js';
 import { formatNoAffectedEvalsMarkdown, formatReportMarkdown, MISSING_RESULTS_BODY, postOrUpdateComment, resolvePrContext, } from '../reporters/github-comment.js';
 const DEFAULT_RESULTS_PATH = path.join('.pathgrade', 'results.json');
 /**
@@ -39,38 +40,17 @@ function resolveCommentId(explicit) {
         return workflow;
     return 'default';
 }
-function isPathgradeReport(value) {
-    if (!value || typeof value !== 'object')
-        return false;
-    const v = value;
-    return ((v.version === 1 || v.version === 2) &&
-        typeof v.overall_pass_rate === 'number' &&
-        (v.status === 'pass' || v.status === 'fail') &&
-        Array.isArray(v.groups) &&
-        (v.version === 1 || (typeof v.overall_mean_reward === 'number'
-            && hasValidAttemptCounts(v)
-            && v.groups.every(group => typeof group.mean_reward === 'number'))));
-}
-function hasValidAttemptCounts(report) {
-    const requested = report.attempts_requested;
-    const completed = report.attempts_completed;
-    return typeof requested === 'number'
-        && Number.isSafeInteger(requested)
-        && requested >= 1
-        && typeof completed === 'number'
-        && Number.isSafeInteger(completed)
-        && completed >= 0
-        && completed <= requested;
-}
 async function loadReport(resolvedPath) {
     if (!(await fs.pathExists(resolvedPath))) {
         throw new Error(`results file not found at ${resolvedPath}`);
     }
     const raw = await fs.readJSON(resolvedPath);
-    if (!isPathgradeReport(raw)) {
+    try {
+        return parsePathgradeReport(raw);
+    }
+    catch {
         throw new Error(`results file at ${resolvedPath} is not a valid pathgrade report`);
     }
-    return raw;
 }
 function printMarkdownAndPassRate(markdown, passRate) {
     // Strip trailing newlines so we control spacing precisely:

package/dist/internal/direct-mcp-v2/acp-author-projector.js

@@ -1,5 +1,7 @@
 import { buildSummary } from '../../tool-events.js';
-import { sanitizePersistenceValue, sanitizeToolEventResult } from '../../tool-event-results.js';
+import { sanitizePersistenceValue, sanitizeToolEventResult, sanitizeUntrustedPersistenceValue, } from '../../tool-event-results.js';
+import { getOriginalMcpInput } from '../../sdk/mcp-event-input.js';
+import { attachLiveMcpInput, attachToolEventSensitiveValues } from '../../sdk/tool-event-secrets.js';
 export const ACP_AUTHOR_TRACE_MAX_CHARS = 64 * 1024;
 function object(value) {
     return value !== null && typeof value === 'object' && !Array.isArray(value) ? value : undefined;
@@ -92,7 +94,7 @@ export function projectAcpAuthorTurn(input) {
             sessionUpdate: current.sessionUpdate,
             title: current.title,
             ...(current.status ? { status: current.status } : {}),
-            ...(current.rawInput ? { rawInput: current.rawInput } : {}),
+            ...(current.rawInput ? { rawInput: sanitizeUntrustedPersistenceValue(current.rawInput, redactionValues) } : {}),
         });
         if (current.status !== 'completed' && current.status !== 'failed') {
             if (current.toolCallId)
@@ -114,7 +116,7 @@ export function projectAcpAuthorTurn(input) {
             toolCalls.set(current.toolCallId, { ...correlated, terminalFingerprint });
         const normalized = input.normalizeToolName(correlated.title);
         const resultText = textValue(correlated.status === 'completed' ? correlated.output : correlated.error);
-        toolEvents.push({
+        const event = {
             action: normalized.action,
             provider: input.provider,
             providerToolName: normalized.providerToolName,
@@ -124,18 +126,21 @@ export function projectAcpAuthorTurn(input) {
             ...(resultText !== undefined ? { result: sanitizeToolEventResult({ content: resultText }, redactionValues) } : {}),
             summary: buildSummary(normalized.action, normalized.providerToolName, correlated.rawInput),
             confidence: normalized.action === 'unknown' ? 'low' : 'high',
-            rawSnippet: JSON.stringify(sanitizePersistenceValue({ title: normalized.providerToolName, status: correlated.status, rawInput: correlated.rawInput }, redactionValues)).slice(0, 2_000),
-        });
+            rawSnippet: JSON.stringify(sanitizeUntrustedPersistenceValue({ title: normalized.providerToolName, status: correlated.status, rawInput: correlated.rawInput }, redactionValues)).slice(0, 2_000),
+        };
+        toolEvents.push(normalized.action === 'mcp_tool_call' && correlated.rawInput
+            ? attachLiveMcpInput(event, correlated.rawInput)
+            : event);
     }
     const assistantMessage = sanitizePersistenceValue(assistantChunks.join(''), redactionValues);
-    const rawTrace = JSON.stringify(sanitizePersistenceValue({
+    const rawTrace = JSON.stringify(sanitizeUntrustedPersistenceValue({
         terminal: { stopReason: input.terminal.stopReason },
         updates: trace,
     }, redactionValues));
     const traceOutput = rawTrace.length > ACP_AUTHOR_TRACE_MAX_CHARS
         ? `${rawTrace.slice(0, ACP_AUTHOR_TRACE_MAX_CHARS - 1)}…`
         : rawTrace;
-    return sanitizePersistenceValue({
+    const result = sanitizePersistenceValue({
         rawOutput: traceOutput,
         traceOutput,
         assistantMessage,
@@ -144,4 +149,12 @@ export function projectAcpAuthorTurn(input) {
         exitCode: input.terminal.stopReason === 'end_turn' && !providerError ? 0 : 1,
         toolEvents,
     }, redactionValues);
+    result.toolEvents.forEach((event, index) => {
+        const originalInput = toolEvents[index] ? getOriginalMcpInput(toolEvents[index]) : undefined;
+        if (event.action === 'mcp_tool_call' && originalInput) {
+            attachLiveMcpInput(event, originalInput);
+            attachToolEventSensitiveValues(event, redactionValues);
+        }
+    });
+    return result;
 }

package/dist/reporters/cli.js

@@ -19,7 +19,8 @@ export async function runCliPreview(resultsDir, opts) {
     console.log(`\n${fmt.bold('pathgrade preview')}  ${fmt.dim(`${entries.length} reports from ${resolved}`)}\n`);
     for (const { file, ...report } of entries) {
         const meanReward = report.mean_reward ?? report.pass_rate ?? 0;
-        const isPass = report.status === undefined ? meanReward >= 0.5 : report.status === 'pass';
+        const isPass = report.run_runner_status === 'fail'
+            ? false : report.status === undefined ? meanReward >= 0.5 : report.status === 'pass';
         const trials = report.trials || [];
         const avgDur = trials.reduce((s, t) => s + (t.duration_ms || 0), 0) / (trials.length || 1);
         const totalTokens = trials.reduce((s, t) => s + (t.input_tokens || 0) + (t.output_tokens || 0) + (t.conversation_input_tokens || 0) + (t.conversation_output_tokens || 0), 0);
@@ -43,6 +44,17 @@ export async function runCliPreview(resultsDir, opts) {
         for (const [label, value] of metrics) {
             console.log(`    ${fmt.dim(label.padEnd(14))} ${fmt.bold(value)}`);
         }
+        if (report.runner_status !== undefined || report.status !== undefined) {
+            console.log(`    ${fmt.dim('Runner assertions'.padEnd(20))} ${gateLabel(report.runner_status ?? report.status)}`);
+        }
+        if (report.run_runner_status !== undefined) {
+            console.log(`    ${fmt.dim('Run assertions'.padEnd(20))} ${gateLabel(report.run_runner_status)}`);
+        }
+        if (report.threshold != null) {
+            const thresholdStatus = report.threshold_status === 'pass' || report.threshold_status === 'fail' ? report.threshold_status : meanReward >= report.threshold ? 'pass' : 'fail';
+            console.log(`    ${fmt.dim('Aggregate threshold'.padEnd(20))} ${gateLabel(thresholdStatus)} — ${(meanReward * 100).toFixed(1)}% ${thresholdStatus === 'pass' ? '>=' : '<'} ${(report.threshold * 100).toFixed(1)}%`);
+        }
+        console.log(`    ${fmt.dim('Overall'.padEnd(20))} ${gateLabel(isPass ? 'pass' : 'fail')}`);
         console.log();
         // ── Trials
         for (const trial of trials) {
@@ -85,6 +97,10 @@ export async function runCliPreview(resultsDir, opts) {
             }
             console.log();
         }
+        if (trials.some((trial) => trial.scorer_results?.some(scorer => scorer.status === 'skipped'))) {
+            console.log(`    ${fmt.dim('Skipped scorers keep their declared weight and contribute zero.')}`);
+            console.log();
+        }
         // ── LLM scorer details
         const hasLlm = trials.some((t) => t.scorer_results?.some((g) => g.scorer_type === 'llm_rubric'));
         if (hasLlm) {
@@ -120,6 +136,9 @@ export async function runCliPreview(resultsDir, opts) {
         console.log();
     }
 }
+function gateLabel(status) {
+    return status === 'pass' ? fmt.pass('PASS') : fmt.fail('FAIL');
+}
 function formatPassAtK(value, reason) {
     if (typeof value === 'number')
         return `${(value * 100).toFixed(1)}% (legacy v1)`;

package/dist/reporters/github-comment.js

@@ -83,15 +83,22 @@ export function formatReportMarkdown(report, opts) {
     }
     if (report.threshold != null) {
         lines.push('');
-        lines.push(`Threshold: ${pct(report.threshold)} — ${report.status.toUpperCase()}`);
+        lines.push(`**Runner assertions:** ${(report.runner_status ?? report.status).toUpperCase()}`);
+        lines.push(`**Aggregate threshold:** ${(report.threshold_status ?? (meanReward >= report.threshold ? 'pass' : 'fail')).toUpperCase()} — ${pct(meanReward)} ${meanReward >= report.threshold ? '>=' : '<'} ${pct(report.threshold)}`);
+        lines.push(`**Overall:** ${report.status.toUpperCase()}`);
+    }
+    else {
+        lines.push('');
+        lines.push(`**Runner assertions:** ${(report.runner_status ?? report.status).toUpperCase()}`);
+        lines.push(`**Overall:** ${report.status.toUpperCase()}`);
     }
     lines.push('');
-    lines.push('| Group | Mean reward | Success rate | pass@k | Skills | Avg duration |');
-    lines.push('|---|---|---|---|---|---|');
+    lines.push('| Group | Status | Mean reward | Success rate | pass@k | Skills | Avg duration |');
+    lines.push('|---|---|---|---|---|---|---|');
     for (const group of report.groups) {
         const skills = group.skills_used.length > 0 ? group.skills_used.join(', ') : '—';
         const avg = computeAvgDuration(group.trials);
-        lines.push(`| ${escapeTableCell(group.task)} | ${pct(group.mean_reward ?? group.pass_rate ?? 0)} | ${group.success_rate === undefined ? '—' : pct(group.success_rate)} | ${escapeTableCell(formatPassAtK(group.pass_at_k, group.pass_at_k_unavailable_reason))} | ${escapeTableCell(skills)} | ${durationSeconds(avg)} |`);
+        lines.push(`| ${escapeTableCell(group.task)} | ${(group.status ?? (group.pass_rate === 1 ? 'pass' : 'fail')).toUpperCase()} | ${pct(group.mean_reward ?? group.pass_rate ?? 0)} | ${group.success_rate === undefined ? '—' : pct(group.success_rate)} | ${escapeTableCell(formatPassAtK(group.pass_at_k, group.pass_at_k_unavailable_reason))} | ${escapeTableCell(skills)} | ${durationSeconds(avg)} |`);
     }
     if (report.selection) {
         lines.push('');
@@ -320,10 +327,14 @@ function formatGroupDetails(group) {
         out.push(`reward: **${trial.reward === undefined ? 'n/a' : trial.reward.toFixed(2)}**  |  duration: ${durationSeconds(trial.duration_ms)}  |  completion: \`${reason}\``);
         if (trial.scorer_results.length > 0) {
             out.push('');
-            out.push('| Scorer | Score | Weight | Details |');
-            out.push('|---|---|---|---|');
+            out.push('| Scorer | Status | Score | Weight | Details |');
+            out.push('|---|---|---|---|---|');
             for (const s of trial.scorer_results) {
-                out.push(`| ${escapeTableCell(s.scorer_type)} | ${s.score.toFixed(2)} | ${s.weight.toFixed(2)} | ${escapeTableCell(s.details ?? '')} |`);
+                out.push(`| ${escapeTableCell(s.scorer_type)} | ${(s.status ?? 'ok').toUpperCase()} | ${s.score.toFixed(2)} | ${s.weight.toFixed(2)} | ${escapeTableCell(s.details ?? '')} |`);
+            }
+            if (trial.scorer_results.some(scorer => scorer.status === 'skipped')) {
+                out.push('');
+                out.push('Skipped scorers keep their declared weight and contribute zero.');
             }
         }
         const warnings = trial.diagnostics?.warnings ?? [];

package/dist/reporters/loader.d.ts

@@ -3,6 +3,10 @@ export interface LoadedReport extends EvalReport {
     file: string;
     timestamp?: string;
     status?: 'pass' | 'fail';
+    runner_status?: 'pass' | 'fail';
+    run_runner_status?: 'pass' | 'fail';
+    threshold_status?: 'pass' | 'fail' | 'not_configured';
+    threshold?: number;
 }
 export declare function loadReports(resultsDir: string, opts?: {
     skipTraces?: boolean;

package/dist/reporters/loader.js

@@ -34,11 +34,18 @@ export async function loadReports(resultsDir, opts) {
             if (raw.version !== 1 && raw.version !== 2)
                 continue;
             for (const group of raw.groups) {
+                const runnerStatus = groupRunnerStatus(group);
+                const thresholdStatus = groupThresholdStatus(raw, group);
                 const report = {
                     file,
                     timestamp: raw.timestamp,
+                    threshold: raw.threshold,
                     ...group,
-                    status: groupStatus(raw, group),
+                    runner_status: runnerStatus,
+                    ...(raw.runner_status === 'pass' || raw.runner_status === 'fail'
+                        ? { run_runner_status: raw.runner_status } : {}),
+                    threshold_status: thresholdStatus,
+                    status: runnerStatus === 'fail' || thresholdStatus === 'fail' ? 'fail' : 'pass',
                 };
                 if (!opts?.skipTraces && group.trace_file) {
                     await hydrateTraces(report, group.trace_file, resolved);
@@ -50,15 +57,25 @@ export async function loadReports(resultsDir, opts) {
     }
     return results;
 }
-function groupStatus(raw, group) {
+function groupRunnerStatus(group) {
+    if (group.runner_status === 'pass' || group.runner_status === 'fail')
+        return group.runner_status;
+    const outcomes = Array.isArray(group.trials)
+        ? group.trials.map((trial) => trial.runner_outcome).filter((outcome) => outcome !== undefined)
+        : [];
+    if (outcomes.length > 0) {
+        return outcomes.every((outcome) => outcome === 'passed') ? 'pass' : 'fail';
+    }
     if (group.status === 'pass' || group.status === 'fail')
         return group.status;
     const meanReward = group.mean_reward ?? group.pass_rate ?? 0;
-    if (typeof raw.threshold === 'number')
-        return meanReward >= raw.threshold ? 'pass' : 'fail';
-    const outcomes = Array.isArray(group.trials) ? group.trials.map((trial) => trial.runner_outcome) : [];
-    if (outcomes.length > 0 && outcomes.every((outcome) => (outcome === 'passed' || outcome === 'failed' || outcome === 'not-run'))) {
-        return outcomes.every((outcome) => outcome === 'passed') ? 'pass' : 'fail';
-    }
-    return meanReward >= 0.5 ? 'pass' : 'fail';
+    return meanReward === 1 ? 'pass' : 'fail';
+}
+function groupThresholdStatus(raw, group) {
+    if (group.threshold_status === 'pass' || group.threshold_status === 'fail'
+        || group.threshold_status === 'not_configured')
+        return group.threshold_status;
+    if (typeof raw.threshold !== 'number')
+        return 'not_configured';
+    return (group.mean_reward ?? group.pass_rate ?? 0) >= raw.threshold ? 'pass' : 'fail';
 }

package/dist/reporting/core.js

@@ -52,9 +52,12 @@ export function buildPathgradeReport(input) {
         .map(attempt => attempt.score)
         .filter((score) => score !== undefined))));
     const overallMeanReward = average(scores);
-    const status = input.threshold != null
-        ? (overallMeanReward >= input.threshold ? 'pass' : 'fail')
-        : (reportableGroups.every(group => group.cases.every(testCase => testCase.state === 'passed')) ? 'pass' : 'fail');
+    const runnerStatus = (input.runStatus === undefined || input.runStatus === 'completed')
+        && reportableGroups.every(group => group.cases.every(testCase => (testCase.state !== 'failed'
+            && testCase.attempts.every(attempt => attempt.outcome.kind === 'passed')))) ? 'pass' : 'fail';
+    const thresholdStatus = input.threshold == null
+        ? 'not_configured'
+        : overallMeanReward >= input.threshold ? 'pass' : 'fail';
     return {
         report: {
             version: 2,
@@ -64,7 +67,9 @@ export function buildPathgradeReport(input) {
             attempts_completed: attemptsCompleted,
             overall_mean_reward: overallMeanReward,
             overall_pass_rate: overallMeanReward,
-            status,
+            runner_status: runnerStatus,
+            threshold_status: thresholdStatus,
+            status: runnerStatus === 'fail' || thresholdStatus === 'fail' ? 'fail' : 'pass',
             groups: consolidatedGroups,
             ...(input.selection ? { selection: input.selection } : {}),
         },
@@ -185,15 +190,20 @@ function buildEvalReport(groupName, cases, run) {
         .filter((score) => score !== undefined);
     const meanReward = average(rewards);
     const metrics = binaryMetrics(cases, run);
+    const runnerStatus = cases.every(testCase => (testCase.state !== 'failed'
+        && testCase.attempts.every(attempt => attempt.outcome.kind === 'passed'))) ? 'pass' : 'fail';
+    const thresholdStatus = run.threshold == null
+        ? 'not_configured'
+        : meanReward >= run.threshold ? 'pass' : 'fail';
     const skills = new Set();
     for (const trial of trials)
         for (const skill of trial.skills_used ?? [])
             skills.add(skill);
     return {
         task: groupName,
-        status: run.threshold != null
-            ? (meanReward >= run.threshold ? 'pass' : 'fail')
-            : (cases.every(testCase => testCase.state === 'passed') ? 'pass' : 'fail'),
+        runner_status: runnerStatus,
+        threshold_status: thresholdStatus,
+        status: runnerStatus === 'fail' || thresholdStatus === 'fail' ? 'fail' : 'pass',
         mean_reward: meanReward,
         ...(metrics.successRate !== undefined ? { success_rate: metrics.successRate } : {}),
         ...(metrics.passAtK ? {

package/dist/reporting/report-parser.js

@@ -6,18 +6,44 @@ export function parsePathgradeReport(value) {
         || (value.version === 2 && (typeof value.overall_mean_reward !== 'number' || !validAttemptCounts(value)))
         || !optionalNumber(value.threshold)
         || (value.status !== 'pass' && value.status !== 'fail')
+        || !optionalGateStatus(value.runner_status)
+        || !optionalThresholdStatus(value.threshold_status)
         || (value.run_kind !== undefined && value.run_kind !== 'evaluation' && value.run_kind !== 'no-affected')
         || !Array.isArray(value.groups)
         || !value.groups.every(group => isReportGroup(group, value.version))
         || !isSelection(value.selection)) {
         throw new Error('PathGrade results.json is missing or has an unsupported schema');
     }
-    return value;
+    const threshold = value.threshold;
+    const groups = value.groups.map(group => {
+        const runnerStatus = deriveRunnerStatus(group);
+        const thresholdStatus = deriveThresholdStatus(group.mean_reward ?? group.pass_rate, threshold);
+        return {
+            ...group,
+            runner_status: runnerStatus,
+            threshold_status: thresholdStatus,
+            status: composeStatus(runnerStatus, thresholdStatus),
+        };
+    });
+    const legacyFailure = value.runner_status === undefined && value.status === 'fail' && groups.length === 0;
+    const runnerStatus = value.runner_status !== 'fail' && !legacyFailure
+        && (groups.length > 0 || value.runner_status === 'pass' || value.status === 'pass')
+        && groups.every(group => group.runner_status === 'pass') ? 'pass' : 'fail';
+    const thresholdStatus = deriveThresholdStatus(value.overall_mean_reward ?? value.overall_pass_rate, threshold);
+    return {
+        ...value,
+        groups,
+        runner_status: runnerStatus,
+        threshold_status: thresholdStatus,
+        status: composeStatus(runnerStatus, thresholdStatus),
+    };
 }
 function isReportGroup(value, version) {
     return isRecord(value)
         && typeof value.task === 'string'
         && (version === 1 ? legacyMetrics(value) : schemaV2Metrics(value))
+        && optionalGateStatus(value.runner_status)
+        && optionalThresholdStatus(value.threshold_status)
         && optionalString(value.source_file)
         && typeof value.trace_file === 'string'
         && Array.isArray(value.skills_used)
@@ -26,6 +52,34 @@ function isReportGroup(value, version) {
         && value.trials.every(isTrial)
         && (value.comparison_contract === undefined || isComparisonContract(value.comparison_contract));
 }
+function deriveRunnerStatus(value) {
+    if (value.runner_status === 'fail')
+        return 'fail';
+    const outcomes = value.trials
+        .map(trial => trial.runner_outcome)
+        .filter(outcome => outcome !== undefined);
+    if (outcomes.length > 0)
+        return outcomes.every(outcome => outcome === 'passed') ? 'pass' : 'fail';
+    if (value.runner_status === 'pass')
+        return value.runner_status;
+    if (value.status === 'pass' || value.status === 'fail')
+        return value.status;
+    return value.pass_rate === 1 ? 'pass' : 'fail';
+}
+function deriveThresholdStatus(reward, threshold) {
+    if (threshold === undefined)
+        return 'not_configured';
+    return typeof reward === 'number' && reward >= threshold ? 'pass' : 'fail';
+}
+function composeStatus(runnerStatus, thresholdStatus) {
+    return runnerStatus === 'fail' || thresholdStatus === 'fail' ? 'fail' : 'pass';
+}
+function optionalGateStatus(value) {
+    return value === undefined || value === 'pass' || value === 'fail';
+}
+function optionalThresholdStatus(value) {
+    return optionalGateStatus(value) || value === 'not_configured';
+}
 function isTrial(value) {
     return isRecord(value)
         && typeof value.trial_id === 'number'
@@ -40,7 +94,8 @@ function isTrial(value) {
         && value.scorer_results.every(scorer => isRecord(scorer)
             && typeof scorer.scorer_type === 'string'
             && typeof scorer.score === 'number'
-            && typeof scorer.weight === 'number');
+            && typeof scorer.weight === 'number'
+            && (scorer.status === undefined || scorer.status === 'ok' || scorer.status === 'error' || scorer.status === 'skipped'));
 }
 function legacyMetrics(value) {
     return typeof value.pass_rate === 'number'

package/dist/reporting/types.d.ts

@@ -1,9 +1,10 @@
 import type { DiagnosticsReport } from '../sdk/diagnostics.js';
 import type { AgentExecutionMetadata, EvaluationResultKind } from '../sdk/types.js';
-import type { AttemptOutcome } from '../runners/model.js';
+import type { AttemptOutcome, RunStatus } from '../runners/model.js';
 import type { PathgradeReport, PathgradeSelectionReport, TrialResult } from '../types.js';
 export type ReportCaseState = 'passed' | 'failed' | 'skipped' | 'pending';
 export interface ReportRunInput {
+    runStatus?: RunStatus;
     threshold?: number;
     selection?: PathgradeSelectionReport;
     attemptsRequested?: number;

package/dist/runners/orchestrator.js

@@ -43,8 +43,8 @@ export async function runWithAdapter(input) {
         if (loadedSelection) {
             built = buildPathgradeReport({
                 threshold: options.threshold,
+                ...reportInput,
                 selection: loadedSelection,
-                groups: reportInput.groups,
             });
         }
         const mode = options.reporterMode ?? 'cli';
@@ -56,14 +56,13 @@ export async function runWithAdapter(input) {
         if (mode === 'browser') {
             await options.openBrowser?.();
         }
-        if (options.threshold != null && built.report.status === 'fail') {
+        if (options.threshold != null && built.report.threshold_status === 'fail') {
             options.onThresholdFailure?.({
                 overallPassRate: built.report.overall_pass_rate,
                 threshold: options.threshold,
             });
-            return runExitCode === 0 ? 1 : runExitCode;
         }
-        return runExitCode;
+        return built.report.status === 'fail' && runExitCode === 0 ? 1 : runExitCode;
     }
     finally {
         await lifecycle.cleanupRun();

package/dist/runners/repeated-invocation.js

@@ -80,13 +80,10 @@ export function withRepeatedAttempts(input) {
                     if (input.config.reporter === 'browser') {
                         await (input.openBrowser ?? openBrowserViewer)();
                     }
-                    if (input.config.ci.threshold != null && built.report.status === 'fail') {
+                    if (input.config.ci.threshold != null && built.report.threshold_status === 'fail') {
                         process.stdout.write(`\n  CI THRESHOLD FAILED  avg score ${built.report.overall_pass_rate.toFixed(3)} < threshold ${input.config.ci.threshold}\n\n`);
-                        resultCode = firstNonzeroExit || 1;
-                    }
-                    else {
-                        resultCode = firstNonzeroExit;
                     }
+                    resultCode = firstNonzeroExit || (built.report.status === 'fail' ? 1 : 0);
                 }
             }
             catch (error) {

package/dist/runners/report-projection.js

@@ -41,6 +41,7 @@ export function projectNormalizedRunSnapshotToReportInput(snapshot, options = {}
     }
     return {
         ...(options.selection ? { selection: options.selection } : {}),
+        runStatus: snapshot.model.run.status,
         attemptsRequested: snapshot.model.run.attemptsRequested ?? 1,
         attemptsCompleted: snapshot.model.run.attemptsCompleted ?? 1,
         groups: [...groups.values()],