@wix/pathgrade 1.0.37 → 1.0.39

package/README.md

@@ -164,6 +164,8 @@ npx pathgrade run
 - **Scorer**: a function or judge that evaluates output or behavior
 - **Evaluation**: the aggregated result of one or more scorers, returned as a score from `0.0` to `1.0`
 
+For protocol-neutral multi-agent graphs, see [Evaluating agent flows](docs/agent-flow-evaluation.md).
+
 ## Scorers
 
 Scorers evaluate the agent's output and behavior. `evaluate()` runs all scorers and computes a weighted average between `0.0` and `1.0`.
@@ -459,7 +461,7 @@ export default {
 
 Pathgrade reads `pathgrade.config.*` for CLI and affected-selection behavior. `runner.adapter` and `--adapter=<name|path>` select the runner; `--adapter` wins over config. `attempts` defaults to `1` and must be a positive integer. Built-in adapters `vitest`, `jest`, and `node-test` support repeated attempts; third-party invocation adapters must advertise `supportsRepeatedAttempts: true` and produce the normalized child snapshot contract.
 
-New reports use schema version 2. They preserve every attempt with `case_id`, `attempt_id`, and `attempt_index`; include the display-safe `runner_outcome` that determined binary success; publish `mean_reward` for partial scores; and publish finite-sample pass@k only for complete binary attempts. Runner assertions, runner diagnostics, and native runner references remain in the normalized run model and are not copied into public report artifacts; evaluation diagnostics retain their existing report behavior. A partial reward or incomplete attempt makes pass@k explicitly unavailable. Version-1 reports remain readable, but Pathgrade no longer computes pass@k or pass^k by pooling heterogeneous cases.
+New reports use schema version 2. They preserve every attempt with `case_id`, `attempt_id`, and `attempt_index`; include the display-safe `runner_outcome` that determined binary success; publish `mean_reward` for partial scores; and publish finite-sample pass@k only for complete binary attempts. `runner_status` and `threshold_status` expose the two gates independently, and canonical `status` passes only when runner assertions pass and any configured threshold passes. Runner assertions, runner diagnostics, and native runner references remain in the normalized run model and are not copied into public report artifacts; evaluation diagnostics retain their existing report behavior. A partial reward or incomplete attempt makes pass@k explicitly unavailable. Version-1 and older version-2 reports remain readable with missing gate fields derived during loading, but Pathgrade no longer computes pass@k or pass^k by pooling heterogeneous cases.
 
 Third-party runner adapters are supported through `@wix/pathgrade/adapter-kit`. Adapter names resolve as follows:
 

package/dist/adapter-kit/index.d.ts

@@ -13,5 +13,5 @@ export { getPathgradeDir } from '../reporters/results-path.js';
 export { printReportSummary } from '../reporters/report-summary.js';
 export { fmt } from '../utils/cli.js';
 export { getCurrentCaseContext, installCaseContextProvider, runWithCaseContext, type CaseContext, type CaseContextProvider, type CaseContextProviderHandle, type CaseContextScope, type CurrentCaseContext, } from '../sdk/case-context.js';
-export { subscribeToEvalResults, type EvalResultObserver, type ResultObserverHandle, type ResultObserverOptions, type ResultObserverOwner, } from '../sdk/result-capture.js';
-export type { Agent, PathgradeTestMeta, RecordedEvalResult, } from '../sdk/types.js';
+export { subscribeToEvalResults, subscribeToEvaluationResults, type EvalResultObserver, type ResultObserverHandle, type ResultObserverOptions, type ResultObserverOwner, type EvaluationResultEvent, type EvaluationResultObserver, } from '../sdk/result-capture.js';
+export type { Agent, AgentFlowTrace, PathgradeTestMeta, RecordedEvalResult, } from '../sdk/types.js';

package/dist/adapter-kit/index.js

@@ -11,4 +11,4 @@ export { getPathgradeDir } from '../reporters/results-path.js';
 export { printReportSummary } from '../reporters/report-summary.js';
 export { fmt } from '../utils/cli.js';
 export { getCurrentCaseContext, installCaseContextProvider, runWithCaseContext, } from '../sdk/case-context.js';
-export { subscribeToEvalResults, } from '../sdk/result-capture.js';
+export { subscribeToEvalResults, subscribeToEvaluationResults, } from '../sdk/result-capture.js';

package/dist/adapters/jest/lifecycle.js

@@ -1,4 +1,4 @@
-import { installCaseContextProvider, subscribeToEvalResults, createRunnerLifecycleHooks, } from '@wix/pathgrade/adapter-kit';
+import { installCaseContextProvider, subscribeToEvaluationResults, createRunnerLifecycleHooks, } from '@wix/pathgrade/adapter-kit';
 import { appendJestMetadata } from './metadata.js';
 import { jestCaseId } from './results.js';
 const metadataByCaseId = new Map();
@@ -58,5 +58,7 @@ function resetJestLifecycleMetadata() {
     metadataByCaseId.clear();
 }
 function defaultSubscribeToResults(callback) {
-    return subscribeToEvalResults(({ result, agent }) => callback({ result, agent }), { owner: 'adapter', key: 'jest-lifecycle' });
+    return subscribeToEvaluationResults(event => callback(event.sourceKind === 'flow'
+        ? { result: event.result, flow: event.flow, case: event.case }
+        : { result: event.result, agent: event.agent, case: event.case }), { owner: 'adapter', key: 'jest-lifecycle' });
 }

package/dist/adapters/node-test/index.js

@@ -2,7 +2,7 @@ import nodeTest from 'node:test';
 import * as fs from 'node:fs';
 import * as path from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { subscribeToEvalResults } from '../../sdk/result-capture.js';
+import { subscribeToEvaluationResults } from '../../sdk/result-capture.js';
 import { createRunnerLifecycleHooks } from '../../runners/lifecycle-hooks.js';
 const lifecycle = createRunnerLifecycleHooks();
 let subscribed = false;
@@ -69,7 +69,9 @@ function installResultCapture() {
     if (subscribed)
         return;
     subscribed = true;
-    subscribeToEvalResults(event => lifecycle.onResult(event), {
+    subscribeToEvaluationResults(event => lifecycle.onResult(event.sourceKind === 'flow'
+        ? { result: event.result, flow: event.flow, case: event.case }
+        : { result: event.result, agent: event.agent, case: event.case }), {
         owner: 'adapter',
         key: 'node-test-lifecycle',
     });

package/dist/agents/claude/sdk-message-projector.js

@@ -22,10 +22,9 @@ const SDK_ERROR_SUBTYPES = [
     'error_max_structured_output_retries',
 ];
 import { TOOL_NAME_MAP, buildSummary, enrichSkillEvents } from '../../tool-events.js';
-import { sanitizePersistenceValue } from '../../tool-event-results.js';
+import { sanitizePersistenceValue, sanitizeUntrustedPersistenceValue } from '../../tool-event-results.js';
 import { attachTurnResultSensitiveValues } from '../../sdk/turn-result-secrets.js';
-import { attachToolEventSensitiveValues } from '../../sdk/tool-event-secrets.js';
-import { attachOriginalMcpInput } from '../../sdk/mcp-event-input.js';
+import { attachLiveMcpInput, attachToolEventSensitiveValues } from '../../sdk/tool-event-secrets.js';
 import { parseClaudeSdkMcpToolName } from './mcp-tool-name.js';
 import { applyObservedToolResult, extractObservedToolResults, } from './tool-results.js';
 export function projectSdkMessages(input) {
@@ -145,7 +144,7 @@ export function projectSdkMessages(input) {
         ...(input.deniedMcpEvents?.all() ?? []),
     ]);
     const finalToolEvents = prependSlashCommandSkillEvent(enriched, input.firstMessage, initSkills).map((event) => attachToolEventSensitiveValues(event, input.sensitiveValues ?? []));
-    const traceOutput = sanitizePersistenceValue(input.messages, input.sensitiveValues)
+    const traceOutput = sanitizeUntrustedPersistenceValue(input.messages, input.sensitiveValues)
         .map((message) => JSON.stringify(message))
         .join('\n');
     const result = {
@@ -216,7 +215,7 @@ function buildToolEvent(block, turnNumber, answerStore, mcpServerNames, deniedMc
             tool: mcpTool.tool,
             status: 'incomplete',
         };
-        return attachOriginalMcpInput({
+        return attachLiveMcpInput({
             action: 'mcp_tool_call',
             provider: 'claude',
             providerToolName: normalizedProviderToolName,

package/dist/agents/claude/tool-permission-bridge.js

@@ -3,6 +3,7 @@ import { createAskUserBridge, } from './ask-user-bridge.js';
 import { parseClaudeSdkMcpToolName } from './mcp-tool-name.js';
 import { canonicalizeJson } from '../../core/canonical-json.js';
 import { buildScriptedMcpApprovalEvent, buildScriptedMcpDeniedCallEvent, } from '../../sdk/scripted-mcp-events.js';
+import { attachLiveMcpInput } from '../../sdk/tool-event-secrets.js';
 export function createClaudeToolPermissionBridge(deps) {
     const askUserBridge = createAskUserBridge({
         askBus: deps.askBus,
@@ -149,5 +150,5 @@ function recordDeniedMcpEvent(opts) {
             input: opts.input,
         })).slice(0, 200),
     };
-    opts.store.record(opts.toolUseId, event);
+    opts.store.record(opts.toolUseId, attachLiveMcpInput(event, opts.input));
 }

package/dist/agents/codex-app-server/item-projection.js

@@ -1,6 +1,6 @@
 import { buildSummary, extractSkillNameFromPath, inferCodexExecAction, } from '../../tool-events.js';
 import { sanitizeToolEventResult } from '../../tool-event-results.js';
-import { attachOriginalMcpInput } from '../../sdk/mcp-event-input.js';
+import { attachLiveMcpInput } from '../../sdk/tool-event-secrets.js';
 export function projectItemIntoTurn(item, turn, sensitiveValues, timing = {}) {
     if (item.type === 'agentMessage') {
         const message = item;
@@ -76,7 +76,7 @@ function projectMcpCall(call, turn, sensitiveValues, timing) {
     const status = call.status === 'completed' && error === undefined
         ? 'completed' : call.status === 'failed' || error !== undefined ? 'error' : 'incomplete';
     const resultContent = call.result === undefined ? undefined : JSON.stringify(call.result);
-    turn.nonAskToolEvents.push(attachOriginalMcpInput({
+    turn.nonAskToolEvents.push(attachLiveMcpInput({
         action: 'mcp_tool_call', provider: 'codex', providerToolName, toolUseId: call.id,
         turnNumber: turn.turnNumber, status, ...projectToolTiming(timing, finiteNumber(call.durationMs)),
         arguments: { ...args, server: call.server, tool: call.tool, status: call.status ?? 'unknown' },

package/dist/agents/codex-app-server/mcp-approval-correlator.js

@@ -1,6 +1,7 @@
 import { decideMcpToolCall, redactMcpSecrets, } from '../../sdk/mcp-safety.js';
 import { canonicalizeJson } from '../../core/canonical-json.js';
 import { buildScriptedMcpApprovalEvent, buildScriptedMcpDeniedCallEvent, } from '../../sdk/scripted-mcp-events.js';
+import { attachLiveMcpInput } from '../../sdk/tool-event-secrets.js';
 function isRecord(value) {
     return !!value && typeof value === 'object' && !Array.isArray(value);
 }
@@ -64,7 +65,7 @@ export function queuePolicyDeniedMcpToolCall(turn, request, decision, rawParams)
 export function buildPolicyDeniedMcpToolEvent(turnNumber, pending, terminal) {
     const args = redactMcpSecrets(pending.request.arguments);
     const providerToolName = `${pending.request.serverName}.${pending.request.toolName}`;
-    return {
+    return attachLiveMcpInput({
         action: 'mcp_tool_call', provider: 'codex', providerToolName,
         ...(terminal ? { toolUseId: terminal.id } : {}),
         turnNumber, status: 'error',
@@ -79,7 +80,7 @@ export function buildPolicyDeniedMcpToolEvent(turnNumber, pending, terminal) {
         },
         summary: `MCP tool ${providerToolName} policy_denied`, confidence: 'high',
         rawSnippet: JSON.stringify(redactMcpSecrets(terminal ? { approval: pending.rawParams, terminal } : pending.rawParams)),
-    };
+    }, pending.request.arguments);
 }
 export function consumeMatchingMcpDenial(turn, serverName, toolName, args) {
     const argsKey = canonicalizeJson(args);

package/dist/agents/opencode.js

@@ -3,12 +3,11 @@ import * as path from 'node:path';
 import fs from 'fs-extra';
 import { BaseAgent, getRuntimeEnv, getWorkspacePath, } from '../types.js';
 import { buildSummary, enrichSkillEvents } from '../tool-events.js';
-import { collectSensitiveEnvValues, sanitizePersistenceValue, sanitizeToolEventResult, } from '../tool-event-results.js';
+import { collectSensitiveEnvValues, sanitizeToolEventResult, sanitizeUntrustedPersistenceValue, } from '../tool-event-results.js';
 import { readStagedMcpServers } from '../providers/mcp-config.js';
 import { removeSandboxRoot } from '../providers/sandbox-lifecycle.js';
 import { attachTurnResultSensitiveValues, cloneTurnResultWithSensitiveValues } from '../sdk/turn-result-secrets.js';
-import { attachOriginalMcpInput } from '../sdk/mcp-event-input.js';
-import { attachToolEventSensitiveValues } from '../sdk/tool-event-secrets.js';
+import { attachLiveMcpInput, attachToolEventSensitiveValues } from '../sdk/tool-event-secrets.js';
 import { currentOpenCodePlatformKey, OPENCODE_RUNTIME_LOCK, } from './opencode/contract.js';
 import { OpenCodeRuntimePolicy, OPENCODE_PERMISSION } from './opencode/runtime-policy.js';
 import { killOpenCodeProcessGroup, registerOpenCodeProcessGroup, unregisterOpenCodeProcessGroup, } from './opencode/process-groups.js';
@@ -217,7 +216,7 @@ export function parseOpenCodeOutput(stdout, processResult, mcpToolNames, sensiti
                 rawSnippet: JSON.stringify({ tool, status: state.status, input }).slice(0, 2_000),
             };
             toolEvents.push(action === 'mcp_tool_call' && input
-                ? attachOriginalMcpInput(toolEvent, input)
+                ? attachLiveMcpInput(toolEvent, input)
                 : toolEvent);
             sanitizedTrace.push({ type, tool, status, input });
             continue;
@@ -256,7 +255,7 @@ export function parseOpenCodeOutput(stdout, processResult, mcpToolNames, sensiti
     if (stepFinishCount === 0)
         throw new Error('OpenCode protocol error: missing step_finish');
     const assistantMessage = textParts.join('');
-    const traceOutput = sanitizePersistenceValue(sanitizedTrace, sensitiveValues)
+    const traceOutput = sanitizeUntrustedPersistenceValue(sanitizedTrace, sensitiveValues)
         .map((event) => JSON.stringify(event))
         .join('\n');
     return {

package/dist/commands/report.js

@@ -20,6 +20,7 @@
 import * as path from 'path';
 import fs from 'fs-extra';
 import { readSidecar } from '../affected/sidecar.js';
+import { parsePathgradeReport } from '../reporting/report-parser.js';
 import { formatNoAffectedEvalsMarkdown, formatReportMarkdown, MISSING_RESULTS_BODY, postOrUpdateComment, resolvePrContext, } from '../reporters/github-comment.js';
 const DEFAULT_RESULTS_PATH = path.join('.pathgrade', 'results.json');
 /**
@@ -39,38 +40,17 @@ function resolveCommentId(explicit) {
         return workflow;
     return 'default';
 }
-function isPathgradeReport(value) {
-    if (!value || typeof value !== 'object')
-        return false;
-    const v = value;
-    return ((v.version === 1 || v.version === 2) &&
-        typeof v.overall_pass_rate === 'number' &&
-        (v.status === 'pass' || v.status === 'fail') &&
-        Array.isArray(v.groups) &&
-        (v.version === 1 || (typeof v.overall_mean_reward === 'number'
-            && hasValidAttemptCounts(v)
-            && v.groups.every(group => typeof group.mean_reward === 'number'))));
-}
-function hasValidAttemptCounts(report) {
-    const requested = report.attempts_requested;
-    const completed = report.attempts_completed;
-    return typeof requested === 'number'
-        && Number.isSafeInteger(requested)
-        && requested >= 1
-        && typeof completed === 'number'
-        && Number.isSafeInteger(completed)
-        && completed >= 0
-        && completed <= requested;
-}
 async function loadReport(resolvedPath) {
     if (!(await fs.pathExists(resolvedPath))) {
         throw new Error(`results file not found at ${resolvedPath}`);
     }
     const raw = await fs.readJSON(resolvedPath);
-    if (!isPathgradeReport(raw)) {
+    try {
+        return parsePathgradeReport(raw);
+    }
+    catch {
         throw new Error(`results file at ${resolvedPath} is not a valid pathgrade report`);
     }
-    return raw;
 }
 function printMarkdownAndPassRate(markdown, passRate) {
     // Strip trailing newlines so we control spacing precisely:

package/dist/internal/direct-mcp-v2/acp-author-projector.js

@@ -1,5 +1,7 @@
 import { buildSummary } from '../../tool-events.js';
-import { sanitizePersistenceValue, sanitizeToolEventResult } from '../../tool-event-results.js';
+import { sanitizePersistenceValue, sanitizeToolEventResult, sanitizeUntrustedPersistenceValue, } from '../../tool-event-results.js';
+import { getOriginalMcpInput } from '../../sdk/mcp-event-input.js';
+import { attachLiveMcpInput, attachToolEventSensitiveValues } from '../../sdk/tool-event-secrets.js';
 export const ACP_AUTHOR_TRACE_MAX_CHARS = 64 * 1024;
 function object(value) {
     return value !== null && typeof value === 'object' && !Array.isArray(value) ? value : undefined;
@@ -92,7 +94,7 @@ export function projectAcpAuthorTurn(input) {
             sessionUpdate: current.sessionUpdate,
             title: current.title,
             ...(current.status ? { status: current.status } : {}),
-            ...(current.rawInput ? { rawInput: current.rawInput } : {}),
+            ...(current.rawInput ? { rawInput: sanitizeUntrustedPersistenceValue(current.rawInput, redactionValues) } : {}),
         });
         if (current.status !== 'completed' && current.status !== 'failed') {
             if (current.toolCallId)
@@ -114,7 +116,7 @@ export function projectAcpAuthorTurn(input) {
             toolCalls.set(current.toolCallId, { ...correlated, terminalFingerprint });
         const normalized = input.normalizeToolName(correlated.title);
         const resultText = textValue(correlated.status === 'completed' ? correlated.output : correlated.error);
-        toolEvents.push({
+        const event = {
             action: normalized.action,
             provider: input.provider,
             providerToolName: normalized.providerToolName,
@@ -124,18 +126,21 @@ export function projectAcpAuthorTurn(input) {
             ...(resultText !== undefined ? { result: sanitizeToolEventResult({ content: resultText }, redactionValues) } : {}),
             summary: buildSummary(normalized.action, normalized.providerToolName, correlated.rawInput),
             confidence: normalized.action === 'unknown' ? 'low' : 'high',
-            rawSnippet: JSON.stringify(sanitizePersistenceValue({ title: normalized.providerToolName, status: correlated.status, rawInput: correlated.rawInput }, redactionValues)).slice(0, 2_000),
-        });
+            rawSnippet: JSON.stringify(sanitizeUntrustedPersistenceValue({ title: normalized.providerToolName, status: correlated.status, rawInput: correlated.rawInput }, redactionValues)).slice(0, 2_000),
+        };
+        toolEvents.push(normalized.action === 'mcp_tool_call' && correlated.rawInput
+            ? attachLiveMcpInput(event, correlated.rawInput)
+            : event);
     }
     const assistantMessage = sanitizePersistenceValue(assistantChunks.join(''), redactionValues);
-    const rawTrace = JSON.stringify(sanitizePersistenceValue({
+    const rawTrace = JSON.stringify(sanitizeUntrustedPersistenceValue({
         terminal: { stopReason: input.terminal.stopReason },
         updates: trace,
     }, redactionValues));
     const traceOutput = rawTrace.length > ACP_AUTHOR_TRACE_MAX_CHARS
         ? `${rawTrace.slice(0, ACP_AUTHOR_TRACE_MAX_CHARS - 1)}…`
         : rawTrace;
-    return sanitizePersistenceValue({
+    const result = sanitizePersistenceValue({
         rawOutput: traceOutput,
         traceOutput,
         assistantMessage,
@@ -144,4 +149,12 @@ export function projectAcpAuthorTurn(input) {
         exitCode: input.terminal.stopReason === 'end_turn' && !providerError ? 0 : 1,
         toolEvents,
     }, redactionValues);
+    result.toolEvents.forEach((event, index) => {
+        const originalInput = toolEvents[index] ? getOriginalMcpInput(toolEvents[index]) : undefined;
+        if (event.action === 'mcp_tool_call' && originalInput) {
+            attachLiveMcpInput(event, originalInput);
+            attachToolEventSensitiveValues(event, redactionValues);
+        }
+    });
+    return result;
 }

package/dist/reporters/cli.js

@@ -19,7 +19,8 @@ export async function runCliPreview(resultsDir, opts) {
     console.log(`\n${fmt.bold('pathgrade preview')}  ${fmt.dim(`${entries.length} reports from ${resolved}`)}\n`);
     for (const { file, ...report } of entries) {
         const meanReward = report.mean_reward ?? report.pass_rate ?? 0;
-        const isPass = report.status === undefined ? meanReward >= 0.5 : report.status === 'pass';
+        const isPass = report.run_runner_status === 'fail'
+            ? false : report.status === undefined ? meanReward >= 0.5 : report.status === 'pass';
         const trials = report.trials || [];
         const avgDur = trials.reduce((s, t) => s + (t.duration_ms || 0), 0) / (trials.length || 1);
         const totalTokens = trials.reduce((s, t) => s + (t.input_tokens || 0) + (t.output_tokens || 0) + (t.conversation_input_tokens || 0) + (t.conversation_output_tokens || 0), 0);
@@ -43,6 +44,17 @@ export async function runCliPreview(resultsDir, opts) {
         for (const [label, value] of metrics) {
             console.log(`    ${fmt.dim(label.padEnd(14))} ${fmt.bold(value)}`);
         }
+        if (report.runner_status !== undefined || report.status !== undefined) {
+            console.log(`    ${fmt.dim('Runner assertions'.padEnd(20))} ${gateLabel(report.runner_status ?? report.status)}`);
+        }
+        if (report.run_runner_status !== undefined) {
+            console.log(`    ${fmt.dim('Run assertions'.padEnd(20))} ${gateLabel(report.run_runner_status)}`);
+        }
+        if (report.threshold != null) {
+            const thresholdStatus = report.threshold_status === 'pass' || report.threshold_status === 'fail' ? report.threshold_status : meanReward >= report.threshold ? 'pass' : 'fail';
+            console.log(`    ${fmt.dim('Aggregate threshold'.padEnd(20))} ${gateLabel(thresholdStatus)} — ${(meanReward * 100).toFixed(1)}% ${thresholdStatus === 'pass' ? '>=' : '<'} ${(report.threshold * 100).toFixed(1)}%`);
+        }
+        console.log(`    ${fmt.dim('Overall'.padEnd(20))} ${gateLabel(isPass ? 'pass' : 'fail')}`);
         console.log();
         // ── Trials
         for (const trial of trials) {
@@ -85,6 +97,10 @@ export async function runCliPreview(resultsDir, opts) {
             }
             console.log();
         }
+        if (trials.some((trial) => trial.scorer_results?.some(scorer => scorer.status === 'skipped'))) {
+            console.log(`    ${fmt.dim('Skipped scorers keep their declared weight and contribute zero.')}`);
+            console.log();
+        }
         // ── LLM scorer details
         const hasLlm = trials.some((t) => t.scorer_results?.some((g) => g.scorer_type === 'llm_rubric'));
         if (hasLlm) {
@@ -120,6 +136,9 @@ export async function runCliPreview(resultsDir, opts) {
         console.log();
     }
 }
+function gateLabel(status) {
+    return status === 'pass' ? fmt.pass('PASS') : fmt.fail('FAIL');
+}
 function formatPassAtK(value, reason) {
     if (typeof value === 'number')
         return `${(value * 100).toFixed(1)}% (legacy v1)`;

package/dist/reporters/github-comment.js

@@ -83,15 +83,22 @@ export function formatReportMarkdown(report, opts) {
     }
     if (report.threshold != null) {
         lines.push('');
-        lines.push(`Threshold: ${pct(report.threshold)} — ${report.status.toUpperCase()}`);
+        lines.push(`**Runner assertions:** ${(report.runner_status ?? report.status).toUpperCase()}`);
+        lines.push(`**Aggregate threshold:** ${(report.threshold_status ?? (meanReward >= report.threshold ? 'pass' : 'fail')).toUpperCase()} — ${pct(meanReward)} ${meanReward >= report.threshold ? '>=' : '<'} ${pct(report.threshold)}`);
+        lines.push(`**Overall:** ${report.status.toUpperCase()}`);
+    }
+    else {
+        lines.push('');
+        lines.push(`**Runner assertions:** ${(report.runner_status ?? report.status).toUpperCase()}`);
+        lines.push(`**Overall:** ${report.status.toUpperCase()}`);
     }
     lines.push('');
-    lines.push('| Group | Mean reward | Success rate | pass@k | Skills | Avg duration |');
-    lines.push('|---|---|---|---|---|---|');
+    lines.push('| Group | Status | Mean reward | Success rate | pass@k | Skills | Avg duration |');
+    lines.push('|---|---|---|---|---|---|---|');
     for (const group of report.groups) {
         const skills = group.skills_used.length > 0 ? group.skills_used.join(', ') : '—';
         const avg = computeAvgDuration(group.trials);
-        lines.push(`| ${escapeTableCell(group.task)} | ${pct(group.mean_reward ?? group.pass_rate ?? 0)} | ${group.success_rate === undefined ? '—' : pct(group.success_rate)} | ${escapeTableCell(formatPassAtK(group.pass_at_k, group.pass_at_k_unavailable_reason))} | ${escapeTableCell(skills)} | ${durationSeconds(avg)} |`);
+        lines.push(`| ${escapeTableCell(group.task)} | ${(group.status ?? (group.pass_rate === 1 ? 'pass' : 'fail')).toUpperCase()} | ${pct(group.mean_reward ?? group.pass_rate ?? 0)} | ${group.success_rate === undefined ? '—' : pct(group.success_rate)} | ${escapeTableCell(formatPassAtK(group.pass_at_k, group.pass_at_k_unavailable_reason))} | ${escapeTableCell(skills)} | ${durationSeconds(avg)} |`);
     }
     if (report.selection) {
         lines.push('');
@@ -320,10 +327,14 @@ function formatGroupDetails(group) {
         out.push(`reward: **${trial.reward === undefined ? 'n/a' : trial.reward.toFixed(2)}**  |  duration: ${durationSeconds(trial.duration_ms)}  |  completion: \`${reason}\``);
         if (trial.scorer_results.length > 0) {
             out.push('');
-            out.push('| Scorer | Score | Weight | Details |');
-            out.push('|---|---|---|---|');
+            out.push('| Scorer | Status | Score | Weight | Details |');
+            out.push('|---|---|---|---|---|');
             for (const s of trial.scorer_results) {
-                out.push(`| ${escapeTableCell(s.scorer_type)} | ${s.score.toFixed(2)} | ${s.weight.toFixed(2)} | ${escapeTableCell(s.details ?? '')} |`);
+                out.push(`| ${escapeTableCell(s.scorer_type)} | ${(s.status ?? 'ok').toUpperCase()} | ${s.score.toFixed(2)} | ${s.weight.toFixed(2)} | ${escapeTableCell(s.details ?? '')} |`);
+            }
+            if (trial.scorer_results.some(scorer => scorer.status === 'skipped')) {
+                out.push('');
+                out.push('Skipped scorers keep their declared weight and contribute zero.');
             }
         }
         const warnings = trial.diagnostics?.warnings ?? [];

package/dist/reporters/loader.d.ts

@@ -3,6 +3,10 @@ export interface LoadedReport extends EvalReport {
     file: string;
     timestamp?: string;
     status?: 'pass' | 'fail';
+    runner_status?: 'pass' | 'fail';
+    run_runner_status?: 'pass' | 'fail';
+    threshold_status?: 'pass' | 'fail' | 'not_configured';
+    threshold?: number;
 }
 export declare function loadReports(resultsDir: string, opts?: {
     skipTraces?: boolean;

package/dist/reporters/loader.js

@@ -34,11 +34,18 @@ export async function loadReports(resultsDir, opts) {
             if (raw.version !== 1 && raw.version !== 2)
                 continue;
             for (const group of raw.groups) {
+                const runnerStatus = groupRunnerStatus(group);
+                const thresholdStatus = groupThresholdStatus(raw, group);
                 const report = {
                     file,
                     timestamp: raw.timestamp,
+                    threshold: raw.threshold,
                     ...group,
-                    status: groupStatus(raw, group),
+                    runner_status: runnerStatus,
+                    ...(raw.runner_status === 'pass' || raw.runner_status === 'fail'
+                        ? { run_runner_status: raw.runner_status } : {}),
+                    threshold_status: thresholdStatus,
+                    status: runnerStatus === 'fail' || thresholdStatus === 'fail' ? 'fail' : 'pass',
                 };
                 if (!opts?.skipTraces && group.trace_file) {
                     await hydrateTraces(report, group.trace_file, resolved);
@@ -50,15 +57,25 @@ export async function loadReports(resultsDir, opts) {
     }
     return results;
 }
-function groupStatus(raw, group) {
+function groupRunnerStatus(group) {
+    if (group.runner_status === 'pass' || group.runner_status === 'fail')
+        return group.runner_status;
+    const outcomes = Array.isArray(group.trials)
+        ? group.trials.map((trial) => trial.runner_outcome).filter((outcome) => outcome !== undefined)
+        : [];
+    if (outcomes.length > 0) {
+        return outcomes.every((outcome) => outcome === 'passed') ? 'pass' : 'fail';
+    }
     if (group.status === 'pass' || group.status === 'fail')
         return group.status;
     const meanReward = group.mean_reward ?? group.pass_rate ?? 0;
-    if (typeof raw.threshold === 'number')
-        return meanReward >= raw.threshold ? 'pass' : 'fail';
-    const outcomes = Array.isArray(group.trials) ? group.trials.map((trial) => trial.runner_outcome) : [];
-    if (outcomes.length > 0 && outcomes.every((outcome) => (outcome === 'passed' || outcome === 'failed' || outcome === 'not-run'))) {
-        return outcomes.every((outcome) => outcome === 'passed') ? 'pass' : 'fail';
-    }
-    return meanReward >= 0.5 ? 'pass' : 'fail';
+    return meanReward === 1 ? 'pass' : 'fail';
+}
+function groupThresholdStatus(raw, group) {
+    if (group.threshold_status === 'pass' || group.threshold_status === 'fail'
+        || group.threshold_status === 'not_configured')
+        return group.threshold_status;
+    if (typeof raw.threshold !== 'number')
+        return 'not_configured';
+    return (group.mean_reward ?? group.pass_rate ?? 0) >= raw.threshold ? 'pass' : 'fail';
 }

package/dist/reporting/comparison-contract.js

@@ -12,7 +12,7 @@ export function buildComparisonContract(input) {
             case: caseIdentity,
             definition: evaluation.evaluationDefinitionKey ?? `position:${index}`,
             scorer: evaluation.scorerRevision,
-            runtime: runtimeIdentity(evaluation.agent ?? evaluation.trial?.agent),
+            runtime: runtimeIdentity(evaluation),
         }));
     });
     const definitionRevision = input.group.sourceRevision
@@ -83,12 +83,26 @@ export function createScorerRevision(scorers) {
         return undefined;
     }
 }
-function runtimeIdentity(agent) {
+function runtimeIdentity(evaluation) {
+    const agent = evaluation.agent ?? evaluation.trial?.agent;
+    if (agent) {
+        return {
+            name: agent.name,
+            model: agent.resolvedModel,
+            transport: agent.transport,
+            interaction: agent.interactionMode,
+        };
+    }
+    const flow = evaluation.trial?.flow_trace;
     return {
-        name: agent?.name,
-        model: agent?.resolvedModel,
-        transport: agent?.transport,
-        interaction: agent?.interactionMode,
+        flow: flow?.completeness.runtimeIdentity === 'complete'
+            && flow.participants.every(participant => participant.runtime) ? {
+            ...(flow.protocol ? { protocol: flow.protocol } : {}),
+            rootParticipantId: flow.rootParticipantId,
+            participants: flow.participants
+                .map(participant => ({ id: participant.id, runtime: participant.runtime }))
+                .toSorted((left, right) => left.id.localeCompare(right.id)),
+        } : undefined,
     };
 }
 function scorerDeclaration(scorer) {

package/dist/reporting/core.js

@@ -30,7 +30,7 @@ export function buildPathgradeReport(input) {
         const traceFile = `traces/${slug(group.groupName)}.json`;
         traces.push({ traceFile, trials: report.trials });
         const strippedTrials = report.trials.map(trial => {
-            const { session_log, conversation, ...rest } = trial;
+            const { session_log, conversation, flow_trace, ...rest } = trial;
             return rest;
         });
         const { trials: _trials, ...rest } = report;
@@ -52,9 +52,12 @@ export function buildPathgradeReport(input) {
         .map(attempt => attempt.score)
         .filter((score) => score !== undefined))));
     const overallMeanReward = average(scores);
-    const status = input.threshold != null
-        ? (overallMeanReward >= input.threshold ? 'pass' : 'fail')
-        : (reportableGroups.every(group => group.cases.every(testCase => testCase.state === 'passed')) ? 'pass' : 'fail');
+    const runnerStatus = (input.runStatus === undefined || input.runStatus === 'completed')
+        && reportableGroups.every(group => group.cases.every(testCase => (testCase.state !== 'failed'
+            && testCase.attempts.every(attempt => attempt.outcome.kind === 'passed')))) ? 'pass' : 'fail';
+    const thresholdStatus = input.threshold == null
+        ? 'not_configured'
+        : overallMeanReward >= input.threshold ? 'pass' : 'fail';
     return {
         report: {
             version: 2,
@@ -64,7 +67,9 @@ export function buildPathgradeReport(input) {
             attempts_completed: attemptsCompleted,
             overall_mean_reward: overallMeanReward,
             overall_pass_rate: overallMeanReward,
-            status,
+            runner_status: runnerStatus,
+            threshold_status: thresholdStatus,
+            status: runnerStatus === 'fail' || thresholdStatus === 'fail' ? 'fail' : 'pass',
             groups: consolidatedGroups,
             ...(input.selection ? { selection: input.selection } : {}),
         },
@@ -185,15 +190,20 @@ function buildEvalReport(groupName, cases, run) {
         .filter((score) => score !== undefined);
     const meanReward = average(rewards);
     const metrics = binaryMetrics(cases, run);
+    const runnerStatus = cases.every(testCase => (testCase.state !== 'failed'
+        && testCase.attempts.every(attempt => attempt.outcome.kind === 'passed'))) ? 'pass' : 'fail';
+    const thresholdStatus = run.threshold == null
+        ? 'not_configured'
+        : meanReward >= run.threshold ? 'pass' : 'fail';
     const skills = new Set();
     for (const trial of trials)
         for (const skill of trial.skills_used ?? [])
             skills.add(skill);
     return {
         task: groupName,
-        status: run.threshold != null
-            ? (meanReward >= run.threshold ? 'pass' : 'fail')
-            : (cases.every(testCase => testCase.state === 'passed') ? 'pass' : 'fail'),
+        runner_status: runnerStatus,
+        threshold_status: thresholdStatus,
+        status: runnerStatus === 'fail' || thresholdStatus === 'fail' ? 'fail' : 'pass',
         mean_reward: meanReward,
         ...(metrics.successRate !== undefined ? { success_rate: metrics.successRate } : {}),
         ...(metrics.passAtK ? {