@wix/pathgrade 1.0.38 → 1.0.40

package/README.md

@@ -178,7 +178,7 @@ Use `check()` for binary requirements, `score()` for partial credit, `judge()` f
 check('tests-pass', async ({ runCommand }) => {
     const { exitCode } = await runCommand('npm test');
     return exitCode === 0;
-});
+}, { revision: { contract: 1 } });
 ```
 
 ### `score()` - Partial credit
@@ -187,7 +187,7 @@ check('tests-pass', async ({ runCommand }) => {
 score('coverage', async ({ runCommand }) => {
     const { stdout } = await runCommand('npx coverage-summary');
     return parseFloat(stdout) / 100;
-});
+}, { revision: { contract: 1 } });
 ```
 
 ### `judge()` - Rubric evaluation
@@ -199,6 +199,7 @@ Was the fix minimal and correct? (0-0.5)`,
 });
 ```
 
+Function-bearing `check()` and `score()` scorers require explicit canonical-JSON `revision` data for authoritative baseline comparisons. A `judge()` with a function-valued `input` requires the same declaration. Imported scorer helpers belong in `comparisonInputs`; closed-over values belong in `revision`.
 Judge scorers also support:
 
 - `retry` for transient judge failures
@@ -440,9 +441,8 @@ Useful details:
 - `pathgrade preview browser` starts a local viewer on `http://localhost:3847`.
 - `pathgrade report` posts or updates a PR comment in GitHub Actions; locally it prints the markdown report and then the numeric pass rate. Provider orchestrators can add a `--details-url`, suppress stale updates with `--expected-head-sha`, and opt into surfaced API failures with `--strict`.
 - `pathgrade validate --affected` is a strict mode for CI: every discovered eval must either live under a `SKILL.md` anchor or export valid `__pathgradeMeta`.
-
+Declare methodology files separately from affected-selection dependencies with `__pathgradeMeta.comparisonInputs`, for example `comparisonInputs: ['evals/fixtures/**', 'evals/scorers/**']`. An empty array explicitly declares a self-contained eval. Pathgrade hashes the raw eval source plus each declared glob, matched repository-relative path, and raw file bytes; `.git` internals and `.pathgrade` outputs are excluded. Missing, invalid, unmatched, escaping, or unreadable inputs suppress numeric deltas. `deps`, `extraDeps`, inferred skill roots, and global triggers affect selection only and never definition identity.
 Run `pathgrade --help` for the full help text.
-
 ## Configuration
 
 ```typescript
@@ -461,7 +461,7 @@ export default {
 
 Pathgrade reads `pathgrade.config.*` for CLI and affected-selection behavior. `runner.adapter` and `--adapter=<name|path>` select the runner; `--adapter` wins over config. `attempts` defaults to `1` and must be a positive integer. Built-in adapters `vitest`, `jest`, and `node-test` support repeated attempts; third-party invocation adapters must advertise `supportsRepeatedAttempts: true` and produce the normalized child snapshot contract.
 
-New reports use schema version 2. They preserve every attempt with `case_id`, `attempt_id`, and `attempt_index`; include the display-safe `runner_outcome` that determined binary success; publish `mean_reward` for partial scores; and publish finite-sample pass@k only for complete binary attempts. Runner assertions, runner diagnostics, and native runner references remain in the normalized run model and are not copied into public report artifacts; evaluation diagnostics retain their existing report behavior. A partial reward or incomplete attempt makes pass@k explicitly unavailable. Version-1 reports remain readable, but Pathgrade no longer computes pass@k or pass^k by pooling heterogeneous cases.
+New reports use schema version 3. They add a versioned task inventory with every selected eval file, collection completeness, stable task keys, and bounded non-scoring reasons. They also preserve every attempt with `case_id`, `attempt_id`, and `attempt_index`; include the display-safe `runner_outcome`; publish `mean_reward`; and publish finite-sample pass@k only for complete binary attempts. `runner_status` and `threshold_status` expose the two gates independently, and canonical `status` passes only when runner assertions pass and any configured threshold passes. Runner assertions, runner diagnostics, native runner references, and full flow traces remain transient or in trace artifacts rather than being copied into consolidated public reports. A partial reward or incomplete attempt makes pass@k explicitly unavailable. Report versions 1 and 2 remain readable, but they cannot be promoted as authoritative task-accounted baselines and must be rerun.
 
 Third-party runner adapters are supported through `@wix/pathgrade/adapter-kit`. Adapter names resolve as follows:
 

package/dist/adapters/jest/results.d.ts

@@ -23,6 +23,7 @@ export declare function normalizeJestRunResults(input: {
     run: AdapterRunHandle;
     results: JestAggregatedResult;
     metadataByCaseId?: Map<string, PathgradeTestMeta[]>;
+    discoveredFiles?: readonly string[];
     cwd?: string;
 }): NormalizedRunSnapshot;
 export declare function jestCaseId(input: {

package/dist/adapters/jest/results.js

@@ -1,20 +1,39 @@
-import { resolveSourceMetadata } from '../../reporting/source-metadata.js';
+import { createSourceMetadataResolver } from '../../reporting/source-metadata.js';
 export function normalizeJestRunResults(input) {
     const runId = `${input.run.adapterName}:run`;
+    const cwd = input.cwd ?? process.cwd();
+    const resolveSourceMetadata = createSourceMetadataResolver(cwd);
     const files = input.results.testResults ?? [];
-    const units = files.map((fileResult, index) => ({
-        id: unitId(index),
-        runId,
-        displayName: fileResult.testFilePath,
-        ...resolveSourceMetadata(fileResult.testFilePath, input.cwd ?? process.cwd()),
-        groupingHints: [{
-                kind: 'source',
-                key: fileResult.testFilePath,
-                label: fileResult.testFilePath,
-                order: index,
-            }],
-        nativeReferences: [{ kind: 'jest-file', id: fileResult.testFilePath }],
-    }));
+    const collection = input.run.status === 'completed'
+        ? { state: 'complete' }
+        : { state: 'incomplete', reason: 'run-incomplete' };
+    const units = [
+        ...files.map((fileResult, index) => ({
+            id: unitId(index),
+            runId,
+            displayName: fileResult.testFilePath,
+            collection,
+            ...resolveSourceMetadata(fileResult.testFilePath),
+            groupingHints: [{
+                    kind: 'source',
+                    key: fileResult.testFilePath,
+                    label: fileResult.testFilePath,
+                    order: index,
+                }],
+            nativeReferences: [{ kind: 'jest-file', id: fileResult.testFilePath }],
+        })),
+        ...(input.discoveredFiles ?? [])
+            .filter(file => !files.some(result => normalizePath(result.testFilePath, resolveSourceMetadata) === file))
+            .map((file, index) => ({
+            id: unitId(files.length + index),
+            runId,
+            displayName: file,
+            collection,
+            ...resolveSourceMetadata(file),
+            groupingHints: [{ kind: 'source', key: file, label: file, order: files.length + index }],
+            nativeReferences: [{ kind: 'jest-file', id: file }],
+        })),
+    ];
     const occurrenceCounts = new Map();
     return {
         version: 1,
@@ -45,6 +64,9 @@ export function normalizeJestRunResults(input) {
         },
     };
 }
+function normalizePath(file, resolveSourceMetadata) {
+    return resolveSourceMetadata(file).sourceFile ?? file.replaceAll('\\', '/');
+}
 function assertionsForFile(fileResult) {
     return fileResult.assertionResults ?? fileResult.testResults ?? [];
 }

package/dist/adapters/jest/runner-adapter.d.ts

@@ -3,6 +3,7 @@ import { type PathgradeTestMeta, RunnerAdapter } from '@wix/pathgrade/adapter-ki
 interface JestRunNative {
     results?: JestAggregatedResult;
     metadataByCaseId?: Map<string, PathgradeTestMeta[]>;
+    discoveredFiles?: string[];
 }
 export declare function createJestAdapter(options?: JestRunNative): RunnerAdapter;
 export declare function createPathgradeAdapter(): RunnerAdapter;

package/dist/adapters/jest/runner-adapter.js

@@ -27,7 +27,10 @@ export function createJestAdapter(options = {}) {
                 adapterName: this.name,
                 status: input.signal?.aborted ? 'cancelled' : 'completed',
                 exitCode: input.signal?.aborted ? 1 : 0,
-                native: options,
+                native: {
+                    ...options,
+                    discoveredFiles: input.discovered.units.flatMap(unit => unit.sourceRef ? [unit.sourceRef] : []),
+                },
             };
         },
         async collectNormalizedRunSnapshot(run) {
@@ -36,6 +39,7 @@ export function createJestAdapter(options = {}) {
                 run,
                 results: native.results ?? { testResults: [] },
                 metadataByCaseId: native.metadataByCaseId,
+                discoveredFiles: native.discoveredFiles,
                 cwd: discoveryCwd,
             });
         },

package/dist/adapters/node-test/runner-adapter.js

@@ -37,7 +37,7 @@ export function createNodeTestAdapter() {
                     adapterName: this.name,
                     status: 'completed',
                     exitCode: 0,
-                    native: { resultsPath, resultsDir, cwd },
+                    native: { resultsPath, resultsDir, cwd, discoveredFiles: files },
                 };
             }
             const exitCode = await spawnNodeTest({
@@ -54,7 +54,7 @@ export function createNodeTestAdapter() {
                 adapterName: this.name,
                 status: input.signal?.aborted ? 'cancelled' : (exitCode === 0 ? 'completed' : 'failed'),
                 exitCode,
-                native: { resultsPath, resultsDir, cwd },
+                native: { resultsPath, resultsDir, cwd, discoveredFiles: files },
             };
         },
         async collectNormalizedRunSnapshot(run) {
@@ -71,7 +71,7 @@ export function createNodeTestAdapter() {
                 return buildNormalizedRunSnapshotFromReportGroups(run, Array.from(groupMap.entries()).map(([groupName, groupedCases]) => ({
                     groupName,
                     cases: groupedCases,
-                })), { cwd: native.cwd });
+                })), { cwd: native.cwd, discoveredFiles: native.discoveredFiles });
             }
             finally {
                 if (native.resultsDir)
@@ -104,10 +104,11 @@ function readNodeTestRunNative(run) {
         && run.native !== null
         && typeof run.native.resultsPath === 'string'
         && typeof run.native.resultsDir === 'string'
-        && typeof run.native.cwd === 'string') {
+        && typeof run.native.cwd === 'string'
+        && Array.isArray(run.native.discoveredFiles)) {
         return run.native;
     }
-    return { resultsPath: '', resultsDir: '', cwd: process.cwd() };
+    return { resultsPath: '', resultsDir: '', cwd: process.cwd(), discoveredFiles: [] };
 }
 async function readCases(resultsPath) {
     if (!resultsPath || !(await fs.pathExists(resultsPath)))

package/dist/affected/meta.d.ts

@@ -17,10 +17,13 @@
 export interface ParsedMeta {
     deps?: string[];
     extraDeps?: string[];
+    comparisonInputs?: string[];
     alwaysRun?: boolean;
 }
 /**
  * Parse `__pathgradeMeta` from an eval file's AST. Returns `null` if the
  * export is not present.
  */
-export declare function parsePathgradeMeta(evalFile: string): ParsedMeta | null;
+export declare function parsePathgradeMeta(evalFile: string, options?: {
+    validateComparisonInputs?: boolean;
+}): ParsedMeta | null;

package/dist/affected/meta.js

@@ -21,14 +21,14 @@ import picomatch from 'picomatch';
  * Parse `__pathgradeMeta` from an eval file's AST. Returns `null` if the
  * export is not present.
  */
-export function parsePathgradeMeta(evalFile) {
+export function parsePathgradeMeta(evalFile, options = {}) {
     const source = fs.readFileSync(evalFile, 'utf-8');
     const sourceFile = ts.createSourceFile(evalFile, source, ts.ScriptTarget.Latest, 
     /* setParentNodes */ true, ts.ScriptKind.TS);
     for (const stmt of sourceFile.statements) {
         const initializer = findMetaInitializer(stmt);
         if (initializer) {
-            return extractMeta(initializer, evalFile);
+            return extractMeta(initializer, evalFile, options);
         }
     }
     return null;
@@ -50,7 +50,7 @@ function findMetaInitializer(stmt) {
     }
     return null;
 }
-function extractMeta(expr, evalFile) {
+function extractMeta(expr, evalFile, options) {
     // Optionally unwrap a type assertion: `{...} as PathgradeMeta` / `<PathgradeMeta>{...}`.
     let node = expr;
     while (ts.isAsExpression(node) || ts.isTypeAssertionExpression(node) || ts.isSatisfiesExpression(node)) {
@@ -64,7 +64,9 @@ function extractMeta(expr, evalFile) {
         if (!ts.isPropertyAssignment(prop))
             continue;
         const name = propertyKeyName(prop.name);
-        if (name === 'deps' || name === 'extraDeps') {
+        if (name === 'deps' || name === 'extraDeps' || name === 'comparisonInputs') {
+            if (name === 'comparisonInputs' && options.validateComparisonInputs === false)
+                continue;
             const globs = extractStringArray(prop.initializer, evalFile, name);
             for (const g of globs)
                 validateGlob(g, evalFile, name);
@@ -89,7 +91,9 @@ function validateGlob(glob, evalFile, field) {
     if (glob.length === 0) {
         throw new Error(`${evalFile}: __pathgradeMeta.${field} contains an empty glob.`);
     }
-    if (glob.startsWith('../') || glob.includes('/../') || glob === '..') {
+    const normalized = glob.replaceAll('\\', '/');
+    if (normalized.startsWith('/') || /^[A-Za-z]:\//.test(normalized)
+        || normalized.startsWith('../') || normalized.includes('/../') || normalized === '..') {
         throw new Error(`${evalFile}: __pathgradeMeta.${field} entry "${glob}" escapes the repo root — ` +
             `globs must be repo-root-relative.`);
     }

package/dist/affected/select.js

@@ -44,7 +44,7 @@ export function selectAffected(input) {
     for (const evalFile of evalFiles) {
         const absEval = path.resolve(repoRoot, evalFile);
         const skillRoot = findSkillRoot(absEval, repoRoot);
-        const meta = parsePathgradeMeta(absEval);
+        const meta = parsePathgradeMeta(absEval, { validateComparisonInputs: false });
         // Precedence #2: alwaysRun wins over dep matching.
         if (meta?.alwaysRun === true) {
             selected.push({ file: evalFile, reason: 'always-run' });

package/dist/agents/claude/sdk-message-projector.js

@@ -22,10 +22,9 @@ const SDK_ERROR_SUBTYPES = [
     'error_max_structured_output_retries',
 ];
 import { TOOL_NAME_MAP, buildSummary, enrichSkillEvents } from '../../tool-events.js';
-import { sanitizePersistenceValue } from '../../tool-event-results.js';
+import { sanitizePersistenceValue, sanitizeUntrustedPersistenceValue } from '../../tool-event-results.js';
 import { attachTurnResultSensitiveValues } from '../../sdk/turn-result-secrets.js';
-import { attachToolEventSensitiveValues } from '../../sdk/tool-event-secrets.js';
-import { attachOriginalMcpInput } from '../../sdk/mcp-event-input.js';
+import { attachLiveMcpInput, attachToolEventSensitiveValues } from '../../sdk/tool-event-secrets.js';
 import { parseClaudeSdkMcpToolName } from './mcp-tool-name.js';
 import { applyObservedToolResult, extractObservedToolResults, } from './tool-results.js';
 export function projectSdkMessages(input) {
@@ -145,7 +144,7 @@ export function projectSdkMessages(input) {
         ...(input.deniedMcpEvents?.all() ?? []),
     ]);
     const finalToolEvents = prependSlashCommandSkillEvent(enriched, input.firstMessage, initSkills).map((event) => attachToolEventSensitiveValues(event, input.sensitiveValues ?? []));
-    const traceOutput = sanitizePersistenceValue(input.messages, input.sensitiveValues)
+    const traceOutput = sanitizeUntrustedPersistenceValue(input.messages, input.sensitiveValues)
         .map((message) => JSON.stringify(message))
         .join('\n');
     const result = {
@@ -216,7 +215,7 @@ function buildToolEvent(block, turnNumber, answerStore, mcpServerNames, deniedMc
             tool: mcpTool.tool,
             status: 'incomplete',
         };
-        return attachOriginalMcpInput({
+        return attachLiveMcpInput({
             action: 'mcp_tool_call',
             provider: 'claude',
             providerToolName: normalizedProviderToolName,

package/dist/agents/claude/tool-permission-bridge.js

@@ -3,6 +3,7 @@ import { createAskUserBridge, } from './ask-user-bridge.js';
 import { parseClaudeSdkMcpToolName } from './mcp-tool-name.js';
 import { canonicalizeJson } from '../../core/canonical-json.js';
 import { buildScriptedMcpApprovalEvent, buildScriptedMcpDeniedCallEvent, } from '../../sdk/scripted-mcp-events.js';
+import { attachLiveMcpInput } from '../../sdk/tool-event-secrets.js';
 export function createClaudeToolPermissionBridge(deps) {
     const askUserBridge = createAskUserBridge({
         askBus: deps.askBus,
@@ -149,5 +150,5 @@ function recordDeniedMcpEvent(opts) {
             input: opts.input,
         })).slice(0, 200),
     };
-    opts.store.record(opts.toolUseId, event);
+    opts.store.record(opts.toolUseId, attachLiveMcpInput(event, opts.input));
 }

package/dist/agents/codex-app-server/item-projection.js

@@ -1,6 +1,6 @@
 import { buildSummary, extractSkillNameFromPath, inferCodexExecAction, } from '../../tool-events.js';
 import { sanitizeToolEventResult } from '../../tool-event-results.js';
-import { attachOriginalMcpInput } from '../../sdk/mcp-event-input.js';
+import { attachLiveMcpInput } from '../../sdk/tool-event-secrets.js';
 export function projectItemIntoTurn(item, turn, sensitiveValues, timing = {}) {
     if (item.type === 'agentMessage') {
         const message = item;
@@ -76,7 +76,7 @@ function projectMcpCall(call, turn, sensitiveValues, timing) {
     const status = call.status === 'completed' && error === undefined
         ? 'completed' : call.status === 'failed' || error !== undefined ? 'error' : 'incomplete';
     const resultContent = call.result === undefined ? undefined : JSON.stringify(call.result);
-    turn.nonAskToolEvents.push(attachOriginalMcpInput({
+    turn.nonAskToolEvents.push(attachLiveMcpInput({
         action: 'mcp_tool_call', provider: 'codex', providerToolName, toolUseId: call.id,
         turnNumber: turn.turnNumber, status, ...projectToolTiming(timing, finiteNumber(call.durationMs)),
         arguments: { ...args, server: call.server, tool: call.tool, status: call.status ?? 'unknown' },

package/dist/agents/codex-app-server/mcp-approval-correlator.js

@@ -1,6 +1,7 @@
 import { decideMcpToolCall, redactMcpSecrets, } from '../../sdk/mcp-safety.js';
 import { canonicalizeJson } from '../../core/canonical-json.js';
 import { buildScriptedMcpApprovalEvent, buildScriptedMcpDeniedCallEvent, } from '../../sdk/scripted-mcp-events.js';
+import { attachLiveMcpInput } from '../../sdk/tool-event-secrets.js';
 function isRecord(value) {
     return !!value && typeof value === 'object' && !Array.isArray(value);
 }
@@ -64,7 +65,7 @@ export function queuePolicyDeniedMcpToolCall(turn, request, decision, rawParams)
 export function buildPolicyDeniedMcpToolEvent(turnNumber, pending, terminal) {
     const args = redactMcpSecrets(pending.request.arguments);
     const providerToolName = `${pending.request.serverName}.${pending.request.toolName}`;
-    return {
+    return attachLiveMcpInput({
         action: 'mcp_tool_call', provider: 'codex', providerToolName,
         ...(terminal ? { toolUseId: terminal.id } : {}),
         turnNumber, status: 'error',
@@ -79,7 +80,7 @@ export function buildPolicyDeniedMcpToolEvent(turnNumber, pending, terminal) {
         },
         summary: `MCP tool ${providerToolName} policy_denied`, confidence: 'high',
         rawSnippet: JSON.stringify(redactMcpSecrets(terminal ? { approval: pending.rawParams, terminal } : pending.rawParams)),
-    };
+    }, pending.request.arguments);
 }
 export function consumeMatchingMcpDenial(turn, serverName, toolName, args) {
     const argsKey = canonicalizeJson(args);

package/dist/agents/opencode.js

@@ -3,12 +3,11 @@ import * as path from 'node:path';
 import fs from 'fs-extra';
 import { BaseAgent, getRuntimeEnv, getWorkspacePath, } from '../types.js';
 import { buildSummary, enrichSkillEvents } from '../tool-events.js';
-import { collectSensitiveEnvValues, sanitizePersistenceValue, sanitizeToolEventResult, } from '../tool-event-results.js';
+import { collectSensitiveEnvValues, sanitizeToolEventResult, sanitizeUntrustedPersistenceValue, } from '../tool-event-results.js';
 import { readStagedMcpServers } from '../providers/mcp-config.js';
 import { removeSandboxRoot } from '../providers/sandbox-lifecycle.js';
 import { attachTurnResultSensitiveValues, cloneTurnResultWithSensitiveValues } from '../sdk/turn-result-secrets.js';
-import { attachOriginalMcpInput } from '../sdk/mcp-event-input.js';
-import { attachToolEventSensitiveValues } from '../sdk/tool-event-secrets.js';
+import { attachLiveMcpInput, attachToolEventSensitiveValues } from '../sdk/tool-event-secrets.js';
 import { currentOpenCodePlatformKey, OPENCODE_RUNTIME_LOCK, } from './opencode/contract.js';
 import { OpenCodeRuntimePolicy, OPENCODE_PERMISSION } from './opencode/runtime-policy.js';
 import { killOpenCodeProcessGroup, registerOpenCodeProcessGroup, unregisterOpenCodeProcessGroup, } from './opencode/process-groups.js';
@@ -217,7 +216,7 @@ export function parseOpenCodeOutput(stdout, processResult, mcpToolNames, sensiti
                 rawSnippet: JSON.stringify({ tool, status: state.status, input }).slice(0, 2_000),
             };
             toolEvents.push(action === 'mcp_tool_call' && input
-                ? attachOriginalMcpInput(toolEvent, input)
+                ? attachLiveMcpInput(toolEvent, input)
                 : toolEvent);
             sanitizedTrace.push({ type, tool, status, input });
             continue;
@@ -256,7 +255,7 @@ export function parseOpenCodeOutput(stdout, processResult, mcpToolNames, sensiti
     if (stepFinishCount === 0)
         throw new Error('OpenCode protocol error: missing step_finish');
     const assistantMessage = textParts.join('');
-    const traceOutput = sanitizePersistenceValue(sanitizedTrace, sensitiveValues)
+    const traceOutput = sanitizeUntrustedPersistenceValue(sanitizedTrace, sensitiveValues)
         .map((event) => JSON.stringify(event))
         .join('\n');
     return {

package/dist/commands/report.js

@@ -20,6 +20,7 @@
 import * as path from 'path';
 import fs from 'fs-extra';
 import { readSidecar } from '../affected/sidecar.js';
+import { parsePathgradeReport } from '../reporting/report-parser.js';
 import { formatNoAffectedEvalsMarkdown, formatReportMarkdown, MISSING_RESULTS_BODY, postOrUpdateComment, resolvePrContext, } from '../reporters/github-comment.js';
 const DEFAULT_RESULTS_PATH = path.join('.pathgrade', 'results.json');
 /**
@@ -39,38 +40,16 @@ function resolveCommentId(explicit) {
         return workflow;
     return 'default';
 }
-function isPathgradeReport(value) {
-    if (!value || typeof value !== 'object')
-        return false;
-    const v = value;
-    return ((v.version === 1 || v.version === 2) &&
-        typeof v.overall_pass_rate === 'number' &&
-        (v.status === 'pass' || v.status === 'fail') &&
-        Array.isArray(v.groups) &&
-        (v.version === 1 || (typeof v.overall_mean_reward === 'number'
-            && hasValidAttemptCounts(v)
-            && v.groups.every(group => typeof group.mean_reward === 'number'))));
-}
-function hasValidAttemptCounts(report) {
-    const requested = report.attempts_requested;
-    const completed = report.attempts_completed;
-    return typeof requested === 'number'
-        && Number.isSafeInteger(requested)
-        && requested >= 1
-        && typeof completed === 'number'
-        && Number.isSafeInteger(completed)
-        && completed >= 0
-        && completed <= requested;
-}
 async function loadReport(resolvedPath) {
     if (!(await fs.pathExists(resolvedPath))) {
         throw new Error(`results file not found at ${resolvedPath}`);
     }
-    const raw = await fs.readJSON(resolvedPath);
-    if (!isPathgradeReport(raw)) {
+    try {
+        return parsePathgradeReport(await fs.readJSON(resolvedPath));
+    }
+    catch {
         throw new Error(`results file at ${resolvedPath} is not a valid pathgrade report`);
     }
-    return raw;
 }
 function printMarkdownAndPassRate(markdown, passRate) {
     // Strip trailing newlines so we control spacing precisely:

package/dist/internal/direct-mcp-v2/acp-author-projector.js

@@ -1,5 +1,7 @@
 import { buildSummary } from '../../tool-events.js';
-import { sanitizePersistenceValue, sanitizeToolEventResult } from '../../tool-event-results.js';
+import { sanitizePersistenceValue, sanitizeToolEventResult, sanitizeUntrustedPersistenceValue, } from '../../tool-event-results.js';
+import { getOriginalMcpInput } from '../../sdk/mcp-event-input.js';
+import { attachLiveMcpInput, attachToolEventSensitiveValues } from '../../sdk/tool-event-secrets.js';
 export const ACP_AUTHOR_TRACE_MAX_CHARS = 64 * 1024;
 function object(value) {
     return value !== null && typeof value === 'object' && !Array.isArray(value) ? value : undefined;
@@ -92,7 +94,7 @@ export function projectAcpAuthorTurn(input) {
             sessionUpdate: current.sessionUpdate,
             title: current.title,
             ...(current.status ? { status: current.status } : {}),
-            ...(current.rawInput ? { rawInput: current.rawInput } : {}),
+            ...(current.rawInput ? { rawInput: sanitizeUntrustedPersistenceValue(current.rawInput, redactionValues) } : {}),
         });
         if (current.status !== 'completed' && current.status !== 'failed') {
             if (current.toolCallId)
@@ -114,7 +116,7 @@ export function projectAcpAuthorTurn(input) {
             toolCalls.set(current.toolCallId, { ...correlated, terminalFingerprint });
         const normalized = input.normalizeToolName(correlated.title);
         const resultText = textValue(correlated.status === 'completed' ? correlated.output : correlated.error);
-        toolEvents.push({
+        const event = {
             action: normalized.action,
             provider: input.provider,
             providerToolName: normalized.providerToolName,
@@ -124,18 +126,21 @@ export function projectAcpAuthorTurn(input) {
             ...(resultText !== undefined ? { result: sanitizeToolEventResult({ content: resultText }, redactionValues) } : {}),
             summary: buildSummary(normalized.action, normalized.providerToolName, correlated.rawInput),
             confidence: normalized.action === 'unknown' ? 'low' : 'high',
-            rawSnippet: JSON.stringify(sanitizePersistenceValue({ title: normalized.providerToolName, status: correlated.status, rawInput: correlated.rawInput }, redactionValues)).slice(0, 2_000),
-        });
+            rawSnippet: JSON.stringify(sanitizeUntrustedPersistenceValue({ title: normalized.providerToolName, status: correlated.status, rawInput: correlated.rawInput }, redactionValues)).slice(0, 2_000),
+        };
+        toolEvents.push(normalized.action === 'mcp_tool_call' && correlated.rawInput
+            ? attachLiveMcpInput(event, correlated.rawInput)
+            : event);
     }
     const assistantMessage = sanitizePersistenceValue(assistantChunks.join(''), redactionValues);
-    const rawTrace = JSON.stringify(sanitizePersistenceValue({
+    const rawTrace = JSON.stringify(sanitizeUntrustedPersistenceValue({
         terminal: { stopReason: input.terminal.stopReason },
         updates: trace,
     }, redactionValues));
     const traceOutput = rawTrace.length > ACP_AUTHOR_TRACE_MAX_CHARS
         ? `${rawTrace.slice(0, ACP_AUTHOR_TRACE_MAX_CHARS - 1)}…`
         : rawTrace;
-    return sanitizePersistenceValue({
+    const result = sanitizePersistenceValue({
         rawOutput: traceOutput,
         traceOutput,
         assistantMessage,
@@ -144,4 +149,12 @@ export function projectAcpAuthorTurn(input) {
         exitCode: input.terminal.stopReason === 'end_turn' && !providerError ? 0 : 1,
         toolEvents,
     }, redactionValues);
+    result.toolEvents.forEach((event, index) => {
+        const originalInput = toolEvents[index] ? getOriginalMcpInput(toolEvents[index]) : undefined;
+        if (event.action === 'mcp_tool_call' && originalInput) {
+            attachLiveMcpInput(event, originalInput);
+            attachToolEventSensitiveValues(event, redactionValues);
+        }
+    });
+    return result;
 }

package/dist/reporters/cli.js

@@ -19,7 +19,8 @@ export async function runCliPreview(resultsDir, opts) {
     console.log(`\n${fmt.bold('pathgrade preview')}  ${fmt.dim(`${entries.length} reports from ${resolved}`)}\n`);
     for (const { file, ...report } of entries) {
         const meanReward = report.mean_reward ?? report.pass_rate ?? 0;
-        const isPass = report.status === undefined ? meanReward >= 0.5 : report.status === 'pass';
+        const isPass = report.run_runner_status === 'fail'
+            ? false : report.status === undefined ? meanReward >= 0.5 : report.status === 'pass';
         const trials = report.trials || [];
         const avgDur = trials.reduce((s, t) => s + (t.duration_ms || 0), 0) / (trials.length || 1);
         const totalTokens = trials.reduce((s, t) => s + (t.input_tokens || 0) + (t.output_tokens || 0) + (t.conversation_input_tokens || 0) + (t.conversation_output_tokens || 0), 0);
@@ -43,6 +44,17 @@ export async function runCliPreview(resultsDir, opts) {
         for (const [label, value] of metrics) {
             console.log(`    ${fmt.dim(label.padEnd(14))} ${fmt.bold(value)}`);
         }
+        if (report.runner_status !== undefined || report.status !== undefined) {
+            console.log(`    ${fmt.dim('Runner assertions'.padEnd(20))} ${gateLabel(report.runner_status ?? report.status)}`);
+        }
+        if (report.run_runner_status !== undefined) {
+            console.log(`    ${fmt.dim('Run assertions'.padEnd(20))} ${gateLabel(report.run_runner_status)}`);
+        }
+        if (report.threshold != null) {
+            const thresholdStatus = report.threshold_status === 'pass' || report.threshold_status === 'fail' ? report.threshold_status : meanReward >= report.threshold ? 'pass' : 'fail';
+            console.log(`    ${fmt.dim('Aggregate threshold'.padEnd(20))} ${gateLabel(thresholdStatus)} — ${(meanReward * 100).toFixed(1)}% ${thresholdStatus === 'pass' ? '>=' : '<'} ${(report.threshold * 100).toFixed(1)}%`);
+        }
+        console.log(`    ${fmt.dim('Overall'.padEnd(20))} ${gateLabel(isPass ? 'pass' : 'fail')}`);
         console.log();
         // ── Trials
         for (const trial of trials) {
@@ -85,6 +97,10 @@ export async function runCliPreview(resultsDir, opts) {
             }
             console.log();
         }
+        if (trials.some((trial) => trial.scorer_results?.some(scorer => scorer.status === 'skipped'))) {
+            console.log(`    ${fmt.dim('Skipped scorers keep their declared weight and contribute zero.')}`);
+            console.log();
+        }
         // ── LLM scorer details
         const hasLlm = trials.some((t) => t.scorer_results?.some((g) => g.scorer_type === 'llm_rubric'));
         if (hasLlm) {
@@ -120,6 +136,9 @@ export async function runCliPreview(resultsDir, opts) {
         console.log();
     }
 }
+function gateLabel(status) {
+    return status === 'pass' ? fmt.pass('PASS') : fmt.fail('FAIL');
+}
 function formatPassAtK(value, reason) {
     if (typeof value === 'number')
         return `${(value * 100).toFixed(1)}% (legacy v1)`;

package/dist/reporters/github-comment.js

@@ -83,15 +83,22 @@ export function formatReportMarkdown(report, opts) {
     }
     if (report.threshold != null) {
         lines.push('');
-        lines.push(`Threshold: ${pct(report.threshold)} — ${report.status.toUpperCase()}`);
+        lines.push(`**Runner assertions:** ${(report.runner_status ?? report.status).toUpperCase()}`);
+        lines.push(`**Aggregate threshold:** ${(report.threshold_status ?? (meanReward >= report.threshold ? 'pass' : 'fail')).toUpperCase()} — ${pct(meanReward)} ${meanReward >= report.threshold ? '>=' : '<'} ${pct(report.threshold)}`);
+        lines.push(`**Overall:** ${report.status.toUpperCase()}`);
+    }
+    else {
+        lines.push('');
+        lines.push(`**Runner assertions:** ${(report.runner_status ?? report.status).toUpperCase()}`);
+        lines.push(`**Overall:** ${report.status.toUpperCase()}`);
     }
     lines.push('');
-    lines.push('| Group | Mean reward | Success rate | pass@k | Skills | Avg duration |');
-    lines.push('|---|---|---|---|---|---|');
+    lines.push('| Group | Status | Mean reward | Success rate | pass@k | Skills | Avg duration |');
+    lines.push('|---|---|---|---|---|---|---|');
     for (const group of report.groups) {
         const skills = group.skills_used.length > 0 ? group.skills_used.join(', ') : '—';
         const avg = computeAvgDuration(group.trials);
-        lines.push(`| ${escapeTableCell(group.task)} | ${pct(group.mean_reward ?? group.pass_rate ?? 0)} | ${group.success_rate === undefined ? '—' : pct(group.success_rate)} | ${escapeTableCell(formatPassAtK(group.pass_at_k, group.pass_at_k_unavailable_reason))} | ${escapeTableCell(skills)} | ${durationSeconds(avg)} |`);
+        lines.push(`| ${escapeTableCell(group.task)} | ${(group.status ?? (group.pass_rate === 1 ? 'pass' : 'fail')).toUpperCase()} | ${pct(group.mean_reward ?? group.pass_rate ?? 0)} | ${group.success_rate === undefined ? '—' : pct(group.success_rate)} | ${escapeTableCell(formatPassAtK(group.pass_at_k, group.pass_at_k_unavailable_reason))} | ${escapeTableCell(skills)} | ${durationSeconds(avg)} |`);
     }
     if (report.selection) {
         lines.push('');
@@ -320,10 +327,14 @@ function formatGroupDetails(group) {
         out.push(`reward: **${trial.reward === undefined ? 'n/a' : trial.reward.toFixed(2)}**  |  duration: ${durationSeconds(trial.duration_ms)}  |  completion: \`${reason}\``);
         if (trial.scorer_results.length > 0) {
             out.push('');
-            out.push('| Scorer | Score | Weight | Details |');
-            out.push('|---|---|---|---|');
+            out.push('| Scorer | Status | Score | Weight | Details |');
+            out.push('|---|---|---|---|---|');
             for (const s of trial.scorer_results) {
-                out.push(`| ${escapeTableCell(s.scorer_type)} | ${s.score.toFixed(2)} | ${s.weight.toFixed(2)} | ${escapeTableCell(s.details ?? '')} |`);
+                out.push(`| ${escapeTableCell(s.scorer_type)} | ${(s.status ?? 'ok').toUpperCase()} | ${s.score.toFixed(2)} | ${s.weight.toFixed(2)} | ${escapeTableCell(s.details ?? '')} |`);
+            }
+            if (trial.scorer_results.some(scorer => scorer.status === 'skipped')) {
+                out.push('');
+                out.push('Skipped scorers keep their declared weight and contribute zero.');
             }
         }
         const warnings = trial.diagnostics?.warnings ?? [];

package/dist/reporters/loader.d.ts

@@ -3,6 +3,10 @@ export interface LoadedReport extends EvalReport {
     file: string;
     timestamp?: string;
     status?: 'pass' | 'fail';
+    runner_status?: 'pass' | 'fail';
+    run_runner_status?: 'pass' | 'fail';
+    threshold_status?: 'pass' | 'fail' | 'not_configured';
+    threshold?: number;
 }
 export declare function loadReports(resultsDir: string, opts?: {
     skipTraces?: boolean;