@wix/pathgrade 1.0.39 → 1.0.41

package/dist/reporting/source-metadata.js

@@ -1,7 +1,18 @@
 import { createHash } from 'node:crypto';
 import { readFileSync } from 'node:fs';
+import { readdirSync } from 'node:fs';
 import * as path from 'node:path';
+import picomatch from 'picomatch';
+import { canonicalizeJson } from '../core/canonical-json.js';
+import { parsePathgradeMeta } from '../affected/meta.js';
 export function resolveSourceMetadata(filePath, cwd) {
+    return resolveSourceMetadataWithFiles(filePath, cwd, () => listRepositoryFiles(cwd));
+}
+export function createSourceMetadataResolver(cwd) {
+    let repositoryFiles;
+    return filePath => resolveSourceMetadataWithFiles(filePath, cwd, () => repositoryFiles ??= listRepositoryFiles(cwd));
+}
+function resolveSourceMetadataWithFiles(filePath, cwd, repositoryFiles) {
     if (!filePath)
         return {};
     const absolute = path.resolve(cwd, filePath);
@@ -11,10 +22,64 @@ export function resolveSourceMetadata(filePath, cwd) {
     try {
         return {
             sourceFile: relative,
-            sourceRevision: `sha256:${createHash('sha256').update(readFileSync(absolute)).digest('hex')}`,
+            sourceRevision: digest(readFileSync(absolute)),
+            comparisonInputs: resolveComparisonInputs(absolute, cwd, repositoryFiles),
+        };
+    }
+    catch {
+        return { sourceFile: relative, comparisonInputs: { state: 'invalid', reason: 'unreadable-input' } };
+    }
+}
+function resolveComparisonInputs(evalFile, cwd, repositoryFiles) {
+    let declarations;
+    try {
+        declarations = parsePathgradeMeta(evalFile)?.comparisonInputs;
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : '';
+        return {
+            state: 'invalid',
+            reason: message.includes('escapes the repo root') ? 'outside-repository' : 'malformed-declaration',
+        };
+    }
+    if (declarations === undefined)
+        return { state: 'missing' };
+    const normalized = declarations.map(value => value.replaceAll('\\', '/').normalize('NFC'));
+    const matched = [...new Set(normalized.flatMap(declaration => {
+            const matches = picomatch(declaration, { dot: true });
+            return repositoryFiles().filter(file => matches(file));
+        }))].toSorted();
+    if (normalized.length > 0 && matched.length === 0)
+        return { state: 'invalid', reason: 'no-matches' };
+    try {
+        const files = matched.map(file => ({ path: file, revision: digest(readFileSync(path.join(cwd, file))) }));
+        return {
+            state: 'resolved',
+            declarations: normalized,
+            files,
+            revision: digest(canonicalizeJson({ declarations: normalized, files })),
         };
     }
     catch {
-        return { sourceFile: relative };
+        return { state: 'invalid', reason: 'unreadable-input' };
     }
 }
+function listRepositoryFiles(cwd) {
+    const files = [];
+    const walk = (directory, relative) => {
+        for (const entry of readdirSync(directory, { withFileTypes: true })) {
+            const childRelative = relative ? `${relative}/${entry.name}` : entry.name;
+            if (!relative && (entry.name === '.git' || entry.name === '.pathgrade'))
+                continue;
+            if (entry.isDirectory())
+                walk(path.join(directory, entry.name), childRelative);
+            else if (entry.isFile())
+                files.push(childRelative);
+        }
+    };
+    walk(path.resolve(cwd), '');
+    return files.toSorted();
+}
+function digest(value) {
+    return `sha256:${createHash('sha256').update(value).digest('hex')}`;
+}

package/dist/reporting/types.d.ts

@@ -2,6 +2,7 @@ import type { DiagnosticsReport } from '../sdk/diagnostics.js';
 import type { AgentExecutionMetadata, EvaluationResultKind } from '../sdk/types.js';
 import type { AttemptOutcome, RunStatus } from '../runners/model.js';
 import type { PathgradeReport, PathgradeSelectionReport, TrialResult } from '../types.js';
+import type { ComparisonInputsState, TaskInventory } from './reliability-contract.js';
 export type ReportCaseState = 'passed' | 'failed' | 'skipped' | 'pending';
 export interface ReportRunInput {
     runStatus?: RunStatus;
@@ -9,12 +10,14 @@ export interface ReportRunInput {
     selection?: PathgradeSelectionReport;
     attemptsRequested?: number;
     attemptsCompleted?: number;
+    taskInventory?: TaskInventory;
     groups: ReportGroupInput[];
 }
 export interface ReportGroupInput {
     groupName: string;
     sourceFile?: string;
     sourceRevision?: string;
+    comparisonInputs?: ComparisonInputsState;
     cases: ReportCaseInput[];
 }
 export interface ReportCaseInput {

package/dist/runners/model-builders.d.ts

@@ -3,4 +3,5 @@ import type { AdapterRunHandle } from './adapter.js';
 import type { NormalizedRunSnapshot } from './model.js';
 export declare function buildNormalizedRunSnapshotFromReportGroups(run: AdapterRunHandle, groups: ReportGroupInput[], options?: {
     cwd?: string;
+    discoveredFiles?: readonly string[];
 }): NormalizedRunSnapshot;

package/dist/runners/model-builders.js

@@ -1,6 +1,7 @@
-import { resolveSourceMetadata } from '../reporting/source-metadata.js';
+import { createSourceMetadataResolver } from '../reporting/source-metadata.js';
 export function buildNormalizedRunSnapshotFromReportGroups(run, groups, options = {}) {
     const cwd = options.cwd ?? process.cwd();
+    const resolveSourceMetadata = createSourceMetadataResolver(cwd);
     return {
         version: 1,
         completeness: 'final',
@@ -14,18 +15,41 @@ export function buildNormalizedRunSnapshotFromReportGroups(run, groups, options
                         message: diagnostic.message,
                     })) } : {}),
             },
-            units: groups.map((group, index) => {
-                const source = group.sourceFile && group.sourceRevision
-                    ? { sourceFile: group.sourceFile, sourceRevision: group.sourceRevision }
-                    : resolveSourceMetadata(group.sourceFile ?? group.cases.map(testCase => testCase.filePath ?? testCase.sourceRef).find(Boolean), cwd);
-                return {
-                    id: unitId(group.groupName, index),
+            units: [
+                ...groups.map((group, index) => {
+                    const source = group.sourceFile && group.sourceRevision
+                        ? {
+                            sourceFile: group.sourceFile,
+                            sourceRevision: group.sourceRevision,
+                            ...(group.comparisonInputs ? { comparisonInputs: group.comparisonInputs } : {}),
+                        }
+                        : resolveSourceMetadata(group.sourceFile ?? group.cases.map(testCase => testCase.filePath ?? testCase.sourceRef).find(Boolean));
+                    return {
+                        id: unitId(group.groupName, index),
+                        runId: `${run.adapterName}:run`,
+                        displayName: group.groupName,
+                        collection: run.status === 'completed' || run.status === 'failed'
+                            ? { state: 'complete' }
+                            : { state: 'incomplete', reason: 'run-incomplete' },
+                        ...source,
+                        groupingHints: [{ kind: 'suite', key: group.groupName, label: group.groupName, order: index }],
+                    };
+                }),
+                ...(options.discoveredFiles ?? [])
+                    .filter(file => !groups.some(group => group.sourceFile === file
+                    || group.cases.some(testCase => testCase.filePath === file || testCase.sourceRef === file)))
+                    .map((file, index) => ({
+                    id: unitId(file, groups.length + index),
                     runId: `${run.adapterName}:run`,
-                    displayName: group.groupName,
-                    ...source,
-                    groupingHints: [{ kind: 'suite', key: group.groupName, label: group.groupName, order: index }],
-                };
-            }),
+                    displayName: file,
+                    collection: {
+                        state: 'incomplete',
+                        reason: 'adapter-cannot-prove-completeness',
+                    },
+                    ...resolveSourceMetadata(file),
+                    groupingHints: [{ kind: 'source', key: file, label: file, order: groups.length + index }],
+                })),
+            ],
             cases: groups.flatMap((group, groupIndex) => group.cases.map((testCase, caseIndex) => {
                 const caseId = testCase.caseId ?? `${unitId(group.groupName, groupIndex)}:case-${caseIndex + 1}`;
                 const attemptId = `${caseId}:attempt-1`;

package/dist/runners/model.d.ts

@@ -1,6 +1,7 @@
 import type { DiagnosticsReport } from '../sdk/diagnostics.js';
 import type { TrialResult } from '../types.js';
 import type { CollectionIncompleteReason } from '../reporting/reliability-contract.js';
+import type { ComparisonInputsState } from '../reporting/reliability-contract.js';
 export interface NormalizedRunSnapshot {
     version: 1;
     completeness: SnapshotCompleteness;
@@ -29,6 +30,7 @@ export interface EvalUnitRecord {
     collection?: EvalUnitCollection;
     sourceFile?: string;
     sourceRevision?: string;
+    comparisonInputs?: ComparisonInputsState;
     diagnostics?: Diagnostic[];
     nativeReferences?: NativeReference[];
     groupingHints?: GroupingHint[];

package/dist/runners/report-projection.js

@@ -1,3 +1,4 @@
+import { TASK_INVENTORY_VERSION } from '../reporting/reliability-contract.js';
 import { validateNormalizedRunSnapshot } from './model-validation.js';
 export function projectNormalizedRunSnapshotToReportInput(snapshot, options = {}) {
     const validation = validateNormalizedRunSnapshot(snapshot, { completeness: 'final' });
@@ -20,6 +21,7 @@ export function projectNormalizedRunSnapshotToReportInput(snapshot, options = {}
             groupName: key,
             ...(unit?.sourceFile ? { sourceFile: unit.sourceFile } : {}),
             ...(unit?.sourceRevision ? { sourceRevision: unit.sourceRevision } : {}),
+            ...(unit?.comparisonInputs ? { comparisonInputs: unit.comparisonInputs } : {}),
             cases: [],
         };
         group.cases.push({
@@ -44,9 +46,90 @@ export function projectNormalizedRunSnapshotToReportInput(snapshot, options = {}
         runStatus: snapshot.model.run.status,
         attemptsRequested: snapshot.model.run.attemptsRequested ?? 1,
         attemptsCompleted: snapshot.model.run.attemptsCompleted ?? 1,
+        taskInventory: buildTaskInventory(snapshot, [...groups.values()]),
         groups: [...groups.values()],
     };
 }
+function buildTaskInventory(snapshot, groups) {
+    const unitsBySource = new Map(snapshot.model.units
+        .filter(unit => unit.sourceFile)
+        .map(unit => [unit.sourceFile, unit]));
+    const files = new Map();
+    for (const unit of snapshot.model.units) {
+        if (!unit.sourceFile)
+            continue;
+        const collection = unit.collection ?? {
+            state: 'incomplete',
+            reason: 'adapter-cannot-prove-completeness',
+        };
+        const existing = files.get(unit.sourceFile);
+        if (!existing) {
+            files.set(unit.sourceFile, {
+                tasks: [],
+                completeness: collection.state,
+                ...(collection.state === 'incomplete' ? { reason: collection.reason } : {}),
+            });
+        }
+        else if (collection.state === 'incomplete') {
+            existing.completeness = 'incomplete';
+            existing.reason = collection.reason;
+        }
+    }
+    for (const group of groups) {
+        if (!group.sourceFile)
+            continue;
+        const unit = unitsBySource.get(group.sourceFile);
+        const collection = unit?.collection ?? {
+            state: 'incomplete',
+            reason: 'adapter-cannot-prove-completeness',
+        };
+        const file = files.get(group.sourceFile) ?? {
+            tasks: [],
+            completeness: collection.state,
+            ...(collection.state === 'incomplete' ? { reason: collection.reason } : {}),
+        };
+        file.tasks.push(taskInventoryEntry(group));
+        if (collection.state === 'incomplete') {
+            file.completeness = 'incomplete';
+            file.reason = collection.reason;
+        }
+        files.set(group.sourceFile, file);
+    }
+    return {
+        version: TASK_INVENTORY_VERSION,
+        files: [...files.entries()].toSorted(([left], [right]) => left.localeCompare(right)).map(([eval_file, file]) => ({
+            eval_file,
+            tasks: file.tasks.toSorted((left, right) => left.task_key.localeCompare(right.task_key)),
+            ...(file.completeness === 'complete'
+                ? { completeness: 'complete' }
+                : { completeness: 'incomplete', reason: file.reason ?? 'adapter-cannot-prove-completeness' }),
+        })),
+    };
+}
+function taskInventoryEntry(group) {
+    const states = group.cases.map(testCase => testCase.state);
+    if (states.every(state => state === 'skipped')) {
+        return { task_key: group.groupName, state: 'not-scored', reason: 'skipped' };
+    }
+    if (states.every(state => state === 'pending')) {
+        return { task_key: group.groupName, state: 'not-scored', reason: 'pending' };
+    }
+    if (group.cases.some(testCase => testCase.attempts?.some(attempt => attempt.outcome.kind === 'timed-out'))) {
+        return { task_key: group.groupName, state: 'not-scored', reason: 'timed-out' };
+    }
+    if (group.cases.some(isPromotableCase))
+        return { task_key: group.groupName, state: 'scored' };
+    return { task_key: group.groupName, state: 'not-scored', reason: 'failed-before-evaluation' };
+}
+function isPromotableCase(testCase) {
+    if (testCase.state === 'skipped' || testCase.state === 'pending')
+        return false;
+    const evaluations = testCase.attempts?.flatMap(attempt => attempt.evaluations ?? []) ?? testCase.evaluations;
+    if (evaluations === undefined)
+        return true;
+    return evaluations.some(evaluation => evaluation.score !== undefined
+        && evaluation.resultKind !== 'synthetic_no_evaluation');
+}
 function totalDurationMs(runCase) {
     return runCase.attempts.reduce((sum, attempt) => sum + (attempt.durationMs ?? 0), 0);
 }

package/dist/runners/vitest-adapter.js

@@ -31,11 +31,15 @@ export function createVitestAdapter(options = {}) {
                 native: {
                     testModules: options.testModules ?? [],
                     cwd: discoveryCwd,
+                    discoveredFiles: input.discovered.units.flatMap(unit => unit.sourceRef ? [unit.sourceRef] : []),
                 },
             };
         },
         async collectNormalizedRunSnapshot(run) {
-            return buildNormalizedRunSnapshotFromReportGroups(run, collectVitestReportGroups(readVitestRunNative(run).testModules), { cwd: readVitestRunNative(run).cwd });
+            return buildNormalizedRunSnapshotFromReportGroups(run, collectVitestReportGroups(readVitestRunNative(run).testModules), {
+                cwd: readVitestRunNative(run).cwd,
+                discoveredFiles: readVitestRunNative(run).discoveredFiles,
+            });
         },
     };
 }
@@ -59,7 +63,7 @@ export function collectVitestReportGroups(testModules) {
 function readVitestRunNative(run) {
     if (isVitestRunNative(run.native))
         return run.native;
-    return { testModules: [], cwd: process.cwd() };
+    return { testModules: [], cwd: process.cwd(), discoveredFiles: [] };
 }
 function isVitestRunNative(native) {
     return typeof native === 'object'

package/dist/sdk/evaluate.js

@@ -270,7 +270,7 @@ function createLiveDeterministicContext(ctx) {
                         tool: event.arguments?.tool,
                         status: event.arguments?.status,
                         ...structuredClone(originalInput),
-                    }, ...(mcp ? { mcp } : {}) })
+                    }, redactedArgumentPaths: undefined, ...(mcp ? { mcp } : {}) })
                 : event;
         }),
     };

package/dist/sdk/index.d.ts

@@ -4,7 +4,7 @@ export { AgentCrashError } from './agent-crash.js';
 export { check, score, judge, toolUsage } from './scorers.js';
 export { compileScenario, compileScenarioText } from './scenario-machine-v2.js';
 export { findScenarioCaseMatches, getFinalScenarioState, getScenarioStateTimeline, ScenarioEvidenceError, wasScenarioCaseMatched, } from './scenario-evidence.js';
-export { getMcpToolCall, isMcpToolCall, findMcpToolCalls, getMcpStartupStatus, isMcpStartupStatus, getMcpApproval, isMcpApproval, findMcpApprovals, getMcpInvocation, wasMcpToolInvoked, } from './mcp-evidence.js';
+export { getMcpToolCall, isMcpToolCall, findMcpToolCalls, McpEvidenceUnavailableError, getMcpStartupStatus, isMcpStartupStatus, getMcpApproval, isMcpApproval, findMcpApprovals, getMcpInvocation, wasMcpToolInvoked, } from './mcp-evidence.js';
 export type { McpMockApprovalRule, McpMockJsonValue } from './mcp-mock-approvals.js';
 export type { CompileScenarioResult, EvidenceEnvelope, JsonObject, JsonPrimitive, JsonValue, MatchExpr, MatchPredicate, McpContent, ScenarioArtifact, ScenarioCase, ScenarioDiagnostic, ScenarioEvidence, ScenarioMachineV2, ScenarioRejection, ScenarioSuccess, ScenarioToolError, } from './scenario-machine-v2.js';
 export type { ExpectedScenarioCaseMatch, ScenarioCaseMatchEvidence, ScenarioStateTransitionEvidence, } from './scenario-evidence.js';

package/dist/sdk/index.js

@@ -5,7 +5,7 @@ export { AgentCrashError } from './agent-crash.js';
 export { check, score, judge, toolUsage } from './scorers.js';
 export { compileScenario, compileScenarioText } from './scenario-machine-v2.js';
 export { findScenarioCaseMatches, getFinalScenarioState, getScenarioStateTimeline, ScenarioEvidenceError, wasScenarioCaseMatched, } from './scenario-evidence.js';
-export { getMcpToolCall, isMcpToolCall, findMcpToolCalls, getMcpStartupStatus, isMcpStartupStatus, getMcpApproval, isMcpApproval, findMcpApprovals, getMcpInvocation, wasMcpToolInvoked, } from './mcp-evidence.js';
+export { getMcpToolCall, isMcpToolCall, findMcpToolCalls, McpEvidenceUnavailableError, getMcpStartupStatus, isMcpStartupStatus, getMcpApproval, isMcpApproval, findMcpApprovals, getMcpInvocation, wasMcpToolInvoked, } from './mcp-evidence.js';
 export { decideMcpToolCall, redactMcpSecrets, } from './mcp-safety.js';
 export { evaluate, evaluateFlow, EvalScorerError } from './evaluate.js';
 export { RUN_SNAPSHOT_VERSION, buildRunSnapshot, loadRunSnapshot, SnapshotParseError, SnapshotVersionError, WorkspaceMissingError, } from './snapshots.js';

package/dist/sdk/mcp-evidence.d.ts

@@ -1,4 +1,7 @@
 import type { ToolEvent } from '../tool-events.js';
+export declare class McpEvidenceUnavailableError extends TypeError {
+    readonly name = "McpEvidenceUnavailableError";
+}
 export type McpInvocationResult = {
     serverName: string;
     toolName: string;
@@ -46,6 +49,7 @@ export interface ExpectedMcpToolCall {
     status?: string;
     invocation?: McpInvocationResult['invocation'];
     outcome?: McpInvocationResult['outcome'];
+    /** Throws when persisted redaction makes the requested argument match unknowable. */
     argumentsContaining?: Record<string, unknown>;
 }
 export interface McpStartupStatusEvidence {

package/dist/sdk/mcp-evidence.js

@@ -1,13 +1,21 @@
+const REDACTED_ARGUMENTS_ERROR = "argumentsContaining cannot be evaluated against redacted MCP arguments; use the scorer's ctx.toolEvents with deterministicToolEvidence: 'live'";
+export class McpEvidenceUnavailableError extends TypeError {
+    name = 'McpEvidenceUnavailableError';
+}
 export function getMcpToolCall(event) {
     if (event.action !== 'mcp_tool_call')
         return undefined;
     const args = event.arguments ?? {};
+    const separator = event.providerToolName.indexOf('.');
     const serverName = event.mcp?.serverName
-        ?? (typeof args.server === 'string' ? args.server : undefined);
+        ?? (typeof args.server === 'string' ? args.server : undefined)
+        ?? (separator > 0 ? event.providerToolName.slice(0, separator) : undefined);
     const toolName = event.mcp?.toolName
-        ?? (typeof args.tool === 'string' ? args.tool : undefined);
+        ?? (typeof args.tool === 'string' ? args.tool : undefined)
+        ?? (separator > 0 && separator < event.providerToolName.length - 1
+            ? event.providerToolName.slice(separator + 1) : undefined);
     const argumentStatus = typeof args.status === 'string' ? args.status : undefined;
-    const status = mcpClassificationStatus(event.mcp, argumentStatus) ?? argumentStatus;
+    const status = mcpClassificationStatus(event.mcp, argumentStatus) ?? argumentStatus ?? event.status;
     if (!serverName || !toolName || !status)
         return undefined;
     return {
@@ -151,29 +159,46 @@ function matchesMcpToolCall(call, expected) {
         if (expected.outcome !== undefined && invocation.outcome !== expected.outcome)
             return false;
     }
-    if (expected.argumentsContaining && !containsArguments(call.arguments, expected.argumentsContaining)) {
+    if (expected.argumentsContaining && !containsArguments(call.arguments, expected.argumentsContaining, call.event.redactedArgumentPaths)) {
         return false;
     }
     return true;
 }
-function containsArguments(actual, expected) {
-    return Object.entries(expected).every(([key, expectedValue]) => valuesEqual(actual[key], expectedValue));
+function containsArguments(actual, expected, redactedPaths = []) {
+    const matches = Object.entries(expected).every(([key, expectedValue]) => valuesEqual(actual[key], expectedValue, `/${escapeJsonPointer(key)}`, redactedPaths));
+    return matches;
 }
-function valuesEqual(actual, expected) {
-    if (Object.is(actual, expected))
+function valuesEqual(actual, expected, path, redactedPaths) {
+    if (Object.is(actual, expected)) {
+        assertMcpArgumentAvailable(path, redactedPaths);
         return true;
+    }
     if (Array.isArray(actual) || Array.isArray(expected)) {
-        if (!Array.isArray(actual) || !Array.isArray(expected))
-            return false;
-        if (actual.length !== expected.length)
-            return false;
-        return actual.every((value, index) => valuesEqual(value, expected[index]));
+        if (Array.isArray(actual) && Array.isArray(expected) && actual.length === expected.length) {
+            return actual.every((value, index) => valuesEqual(value, expected[index], `${path}/${index}`, redactedPaths));
+        }
+        return rejectRedactedMismatch(path, redactedPaths);
     }
     if (isRecord(actual) && isRecord(expected)) {
-        return Object.entries(expected).every(([key, value]) => valuesEqual(actual[key], value));
+        return Object.entries(expected).every(([key, value]) => valuesEqual(actual[key], value, `${path}/${escapeJsonPointer(key)}`, redactedPaths));
     }
+    return rejectRedactedMismatch(path, redactedPaths);
+}
+function rejectRedactedMismatch(path, redactedPaths) {
+    assertMcpArgumentAvailable(path, redactedPaths);
     return false;
 }
+function assertMcpArgumentAvailable(path, redactedPaths) {
+    if (redactedPaths.some((redactedPath) => redactedPath === ''
+        || redactedPath === path
+        || redactedPath.startsWith(`${path}/`)
+        || path.startsWith(`${redactedPath}/`))) {
+        throw new McpEvidenceUnavailableError(REDACTED_ARGUMENTS_ERROR);
+    }
+}
+function escapeJsonPointer(value) {
+    return value.replaceAll('~', '~0').replaceAll('/', '~1');
+}
 function isRecord(value) {
     return !!value && typeof value === 'object' && !Array.isArray(value);
 }

package/dist/sdk/run-scorer.js

@@ -1,6 +1,7 @@
 import { runJudgePipeline } from './judge-pipeline.js';
 import { runJudgeSession } from './judge-tool-session.js';
 import { clamp, makeErroredResult, matchesExpectation } from './scorer-utils.js';
+import { McpEvidenceUnavailableError } from './mcp-evidence.js';
 export async function runScorer(scorer, ctx, opts = {}) {
     switch (scorer.type) {
         case 'check':
@@ -56,6 +57,9 @@ async function runCheckScorer(scorer, ctx) {
         };
     }
     catch (error) {
+        if (error instanceof McpEvidenceUnavailableError) {
+            return makeUnavailableEvidenceResult('check', scorer.name, scorer.weight, error);
+        }
         return makeErroredResult('check', scorer.name, scorer.weight, error);
     }
 }
@@ -81,9 +85,15 @@ async function runScoreScorer(scorer, ctx) {
         };
     }
     catch (error) {
+        if (error instanceof McpEvidenceUnavailableError) {
+            return makeUnavailableEvidenceResult('score', scorer.name, scorer.weight, error);
+        }
         return makeErroredResult('score', scorer.name, scorer.weight, error);
     }
 }
+function makeUnavailableEvidenceResult(type, name, weight, error) {
+    return { name, type, score: 0, weight, details: error.message, status: 'ok' };
+}
 async function runToolUsageScorer(scorer, ctx) {
     try {
         const toolEvents = ctx.toolEvents;

package/dist/sdk/scorers.d.ts

@@ -1,15 +1,18 @@
 import type { CheckScorer, CodeJudgeToolName, ScorerContext, JudgeScorer, ScoreScorer, ScoreResult, ToolExpectation, ToolUsageScorer } from './types.js';
+import type { JsonValue } from '../internal/direct-mcp-v2/types.js';
 /**
  * Create a check scorer — a boolean gate that passes (1.0) or fails (0.0).
  */
 export declare function check(name: string, fn: (ctx: ScorerContext) => boolean | Promise<boolean>, opts?: {
     weight?: number;
+    revision?: JsonValue;
 }): CheckScorer;
 /**
  * Create a score scorer — returns a number (0-1) or { score, details } for partial credit.
  */
 export declare function score(name: string, fn: (ctx: ScorerContext) => number | ScoreResult | Promise<number | ScoreResult>, opts?: {
     weight?: number;
+    revision?: JsonValue;
 }): ScoreScorer;
 /**
  * Create a judge scorer — uses an LLM to evaluate against a rubric.
@@ -55,10 +58,12 @@ export declare function judge(name: string, opts: {
     maxRounds?: number;
     /** Anthropic prompt caching. Default: true when tools is set, unchanged otherwise. */
     cacheControl?: boolean;
+    revision?: JsonValue;
 }): JudgeScorer;
 /**
  * Create a tool usage scorer — matches tool events against expectations.
  */
 export declare function toolUsage(name: string, expectations: ToolExpectation[], opts?: {
     weight?: number;
+    revision?: JsonValue;
 }): ToolUsageScorer;

package/dist/sdk/scorers.js

@@ -7,6 +7,7 @@ export function check(name, fn, opts) {
         name,
         weight: opts?.weight ?? 1,
         fn,
+        ...(opts?.revision === undefined ? {} : { revision: opts.revision }),
     };
 }
 /**
@@ -18,6 +19,7 @@ export function score(name, fn, opts) {
         name,
         weight: opts?.weight ?? 1,
         fn,
+        ...(opts?.revision === undefined ? {} : { revision: opts.revision }),
     };
 }
 /**
@@ -65,6 +67,7 @@ export function judge(name, opts) {
         tools,
         maxRounds: opts.maxRounds,
         cacheControl,
+        ...(opts.revision === undefined ? {} : { revision: opts.revision }),
     };
 }
 /**
@@ -76,5 +79,6 @@ export function toolUsage(name, expectations, opts) {
         name,
         weight: opts?.weight ?? 1,
         expectations,
+        ...(opts?.revision === undefined ? {} : { revision: opts.revision }),
     };
 }

package/dist/sdk/tool-event-log.js

@@ -1,5 +1,8 @@
+import { isDeepStrictEqual } from 'node:util';
 import { collectStructuredSensitiveValues, isSensitiveValueScanLimitError, sanitizePersistenceValue, } from '../tool-event-results.js';
 import { attachToolEventSensitiveValues, cloneToolEventWithRuntimeMetadata, getToolEventSensitiveValues, redactToolEventPayload, } from './tool-event-secrets.js';
+import { getOriginalMcpInput } from './mcp-event-input.js';
+const MCP_METADATA_ARGUMENTS = new Set(['server', 'tool', 'status']);
 export function buildToolEventLogEntry(toolEvent, fallbackTimestamp) {
     let event = toolEvent;
     let discoveredValues;
@@ -16,10 +19,37 @@ export function buildToolEventLogEntry(toolEvent, fallbackTimestamp) {
             ...getToolEventSensitiveValues(toolEvent),
             ...discoveredValues,
         ])];
-    const persistedEvent = attachToolEventSensitiveValues(cloneToolEventWithRuntimeMetadata(event, sanitizePersistenceValue(event, sensitiveValues)), sensitiveValues);
+    const sanitizedEvent = sanitizePersistenceValue(event, sensitiveValues);
+    const originalInput = getOriginalMcpInput(event);
+    const originalArguments = originalInput ? {
+        ...event.arguments,
+        ...Object.fromEntries(Object.entries(originalInput).filter(([key]) => !MCP_METADATA_ARGUMENTS.has(key))),
+    } : event.arguments;
+    const redactedArgumentPaths = [...new Set([
+            ...(event.redactedArgumentPaths ?? []),
+            ...collectChangedPaths(originalArguments, sanitizedEvent.arguments),
+        ])];
+    const persistedEvent = attachToolEventSensitiveValues(cloneToolEventWithRuntimeMetadata(event, {
+        ...sanitizedEvent,
+        ...(redactedArgumentPaths.length > 0 ? { redactedArgumentPaths } : {}),
+    }), sensitiveValues);
     return {
         type: 'tool_event',
         timestamp: persistedEvent.startedAt ?? fallbackTimestamp,
         tool_event: persistedEvent,
     };
 }
+function collectChangedPaths(original, sanitized, path = '') {
+    if (isDeepStrictEqual(original, sanitized))
+        return [];
+    if (Array.isArray(original) && Array.isArray(sanitized) && original.length === sanitized.length) {
+        return original.flatMap((value, index) => collectChangedPaths(value, sanitized[index], `${path}/${index}`));
+    }
+    if (isRecord(original) && isRecord(sanitized)) {
+        return Object.keys(original).flatMap((key) => collectChangedPaths(original[key], sanitized[key], `${path}/${key.replaceAll('~', '~0').replaceAll('/', '~1')}`));
+    }
+    return [path];
+}
+function isRecord(value) {
+    return !!value && typeof value === 'object' && !Array.isArray(value);
+}

package/dist/sdk/tool-event-secrets.js

@@ -27,6 +27,7 @@ export function redactToolEventPayload(event) {
     return {
         ...event,
         arguments: { redacted: true },
+        redactedArgumentPaths: [''],
         summary: `${event.action} via ${event.providerToolName}`,
         rawSnippet: '<redacted>',
     };

package/dist/sdk/types.d.ts

@@ -348,6 +348,7 @@ export interface ScorerContext {
     workspace: string;
     log: LogEntry[];
     transcript: string;
+    /** Use this scorer-context field for live MCP arguments; `log` always remains sanitized. */
     toolEvents: import('../tool-events.js').ToolEvent[];
     runCommand: (cmd: string) => Promise<CommandResult>;
     artifacts: SessionArtifacts;
@@ -360,7 +361,7 @@ export interface EvaluateOptions {
     failFast?: boolean;
     llm?: LLMPort;
     onScorerError?: 'skip' | 'zero' | 'fail';
-    /** Evidence visible to deterministic scorers. Live mode is unavailable for snapshot replay. */
+    /** Restores exact MCP arguments only on deterministic scorers' `ctx.toolEvents`; `agent.log` remains sanitized. Live mode is unavailable for snapshot replay. */
     deterministicToolEvidence?: 'persisted' | 'live';
     /** Stable identity for this evaluation definition across separate runs. */
     evaluationDefinitionKey?: string;

package/dist/tool-events.d.ts

@@ -20,6 +20,8 @@ export interface ToolEvent {
     toolUseId?: string;
     turnNumber?: number;
     arguments?: Record<string, unknown>;
+    /** Producer-controlled JSON Pointer paths identifying persisted arguments that were sanitized. */
+    redactedArgumentPaths?: string[];
     /** Lifecycle state observed by PathGrade. Absent on legacy provider events. */
     status?: 'completed' | 'error' | 'incomplete';
     /** Receipt/enforcement-backed classification for canonical MCP calls. */