@wix/pathgrade 1.0.38 → 1.0.40

package/dist/runners/model-builders.js

@@ -1,6 +1,7 @@
-import { resolveSourceMetadata } from '../reporting/source-metadata.js';
+import { createSourceMetadataResolver } from '../reporting/source-metadata.js';
 export function buildNormalizedRunSnapshotFromReportGroups(run, groups, options = {}) {
     const cwd = options.cwd ?? process.cwd();
+    const resolveSourceMetadata = createSourceMetadataResolver(cwd);
     return {
         version: 1,
         completeness: 'final',
@@ -14,18 +15,41 @@ export function buildNormalizedRunSnapshotFromReportGroups(run, groups, options
                         message: diagnostic.message,
                     })) } : {}),
             },
-            units: groups.map((group, index) => {
-                const source = group.sourceFile && group.sourceRevision
-                    ? { sourceFile: group.sourceFile, sourceRevision: group.sourceRevision }
-                    : resolveSourceMetadata(group.sourceFile ?? group.cases.map(testCase => testCase.filePath ?? testCase.sourceRef).find(Boolean), cwd);
-                return {
-                    id: unitId(group.groupName, index),
+            units: [
+                ...groups.map((group, index) => {
+                    const source = group.sourceFile && group.sourceRevision
+                        ? {
+                            sourceFile: group.sourceFile,
+                            sourceRevision: group.sourceRevision,
+                            ...(group.comparisonInputs ? { comparisonInputs: group.comparisonInputs } : {}),
+                        }
+                        : resolveSourceMetadata(group.sourceFile ?? group.cases.map(testCase => testCase.filePath ?? testCase.sourceRef).find(Boolean));
+                    return {
+                        id: unitId(group.groupName, index),
+                        runId: `${run.adapterName}:run`,
+                        displayName: group.groupName,
+                        collection: run.status === 'completed' || run.status === 'failed'
+                            ? { state: 'complete' }
+                            : { state: 'incomplete', reason: 'run-incomplete' },
+                        ...source,
+                        groupingHints: [{ kind: 'suite', key: group.groupName, label: group.groupName, order: index }],
+                    };
+                }),
+                ...(options.discoveredFiles ?? [])
+                    .filter(file => !groups.some(group => group.sourceFile === file
+                    || group.cases.some(testCase => testCase.filePath === file || testCase.sourceRef === file)))
+                    .map((file, index) => ({
+                    id: unitId(file, groups.length + index),
                     runId: `${run.adapterName}:run`,
-                    displayName: group.groupName,
-                    ...source,
-                    groupingHints: [{ kind: 'suite', key: group.groupName, label: group.groupName, order: index }],
-                };
-            }),
+                    displayName: file,
+                    collection: {
+                        state: 'incomplete',
+                        reason: 'adapter-cannot-prove-completeness',
+                    },
+                    ...resolveSourceMetadata(file),
+                    groupingHints: [{ kind: 'source', key: file, label: file, order: groups.length + index }],
+                })),
+            ],
             cases: groups.flatMap((group, groupIndex) => group.cases.map((testCase, caseIndex) => {
                 const caseId = testCase.caseId ?? `${unitId(group.groupName, groupIndex)}:case-${caseIndex + 1}`;
                 const attemptId = `${caseId}:attempt-1`;

package/dist/runners/model.d.ts

@@ -1,6 +1,7 @@
 import type { DiagnosticsReport } from '../sdk/diagnostics.js';
 import type { TrialResult } from '../types.js';
 import type { CollectionIncompleteReason } from '../reporting/reliability-contract.js';
+import type { ComparisonInputsState } from '../reporting/reliability-contract.js';
 export interface NormalizedRunSnapshot {
     version: 1;
     completeness: SnapshotCompleteness;
@@ -29,6 +30,7 @@ export interface EvalUnitRecord {
     collection?: EvalUnitCollection;
     sourceFile?: string;
     sourceRevision?: string;
+    comparisonInputs?: ComparisonInputsState;
     diagnostics?: Diagnostic[];
     nativeReferences?: NativeReference[];
     groupingHints?: GroupingHint[];

package/dist/runners/orchestrator.js

@@ -43,8 +43,8 @@ export async function runWithAdapter(input) {
         if (loadedSelection) {
             built = buildPathgradeReport({
                 threshold: options.threshold,
+                ...reportInput,
                 selection: loadedSelection,
-                groups: reportInput.groups,
             });
         }
         const mode = options.reporterMode ?? 'cli';
@@ -56,14 +56,13 @@ export async function runWithAdapter(input) {
         if (mode === 'browser') {
             await options.openBrowser?.();
         }
-        if (options.threshold != null && built.report.status === 'fail') {
+        if (options.threshold != null && built.report.threshold_status === 'fail') {
             options.onThresholdFailure?.({
                 overallPassRate: built.report.overall_pass_rate,
                 threshold: options.threshold,
             });
-            return runExitCode === 0 ? 1 : runExitCode;
         }
-        return runExitCode;
+        return built.report.status === 'fail' && runExitCode === 0 ? 1 : runExitCode;
     }
     finally {
         await lifecycle.cleanupRun();

package/dist/runners/repeated-invocation.js

@@ -80,13 +80,10 @@ export function withRepeatedAttempts(input) {
                     if (input.config.reporter === 'browser') {
                         await (input.openBrowser ?? openBrowserViewer)();
                     }
-                    if (input.config.ci.threshold != null && built.report.status === 'fail') {
+                    if (input.config.ci.threshold != null && built.report.threshold_status === 'fail') {
                         process.stdout.write(`\n  CI THRESHOLD FAILED  avg score ${built.report.overall_pass_rate.toFixed(3)} < threshold ${input.config.ci.threshold}\n\n`);
-                        resultCode = firstNonzeroExit || 1;
-                    }
-                    else {
-                        resultCode = firstNonzeroExit;
                     }
+                    resultCode = firstNonzeroExit || (built.report.status === 'fail' ? 1 : 0);
                 }
             }
             catch (error) {

package/dist/runners/report-projection.js

@@ -1,3 +1,4 @@
+import { TASK_INVENTORY_VERSION } from '../reporting/reliability-contract.js';
 import { validateNormalizedRunSnapshot } from './model-validation.js';
 export function projectNormalizedRunSnapshotToReportInput(snapshot, options = {}) {
     const validation = validateNormalizedRunSnapshot(snapshot, { completeness: 'final' });
@@ -20,6 +21,7 @@ export function projectNormalizedRunSnapshotToReportInput(snapshot, options = {}
             groupName: key,
             ...(unit?.sourceFile ? { sourceFile: unit.sourceFile } : {}),
             ...(unit?.sourceRevision ? { sourceRevision: unit.sourceRevision } : {}),
+            ...(unit?.comparisonInputs ? { comparisonInputs: unit.comparisonInputs } : {}),
             cases: [],
         };
         group.cases.push({
@@ -41,11 +43,93 @@ export function projectNormalizedRunSnapshotToReportInput(snapshot, options = {}
     }
     return {
         ...(options.selection ? { selection: options.selection } : {}),
+        runStatus: snapshot.model.run.status,
         attemptsRequested: snapshot.model.run.attemptsRequested ?? 1,
         attemptsCompleted: snapshot.model.run.attemptsCompleted ?? 1,
+        taskInventory: buildTaskInventory(snapshot, [...groups.values()]),
         groups: [...groups.values()],
     };
 }
+function buildTaskInventory(snapshot, groups) {
+    const unitsBySource = new Map(snapshot.model.units
+        .filter(unit => unit.sourceFile)
+        .map(unit => [unit.sourceFile, unit]));
+    const files = new Map();
+    for (const unit of snapshot.model.units) {
+        if (!unit.sourceFile)
+            continue;
+        const collection = unit.collection ?? {
+            state: 'incomplete',
+            reason: 'adapter-cannot-prove-completeness',
+        };
+        const existing = files.get(unit.sourceFile);
+        if (!existing) {
+            files.set(unit.sourceFile, {
+                tasks: [],
+                completeness: collection.state,
+                ...(collection.state === 'incomplete' ? { reason: collection.reason } : {}),
+            });
+        }
+        else if (collection.state === 'incomplete') {
+            existing.completeness = 'incomplete';
+            existing.reason = collection.reason;
+        }
+    }
+    for (const group of groups) {
+        if (!group.sourceFile)
+            continue;
+        const unit = unitsBySource.get(group.sourceFile);
+        const collection = unit?.collection ?? {
+            state: 'incomplete',
+            reason: 'adapter-cannot-prove-completeness',
+        };
+        const file = files.get(group.sourceFile) ?? {
+            tasks: [],
+            completeness: collection.state,
+            ...(collection.state === 'incomplete' ? { reason: collection.reason } : {}),
+        };
+        file.tasks.push(taskInventoryEntry(group));
+        if (collection.state === 'incomplete') {
+            file.completeness = 'incomplete';
+            file.reason = collection.reason;
+        }
+        files.set(group.sourceFile, file);
+    }
+    return {
+        version: TASK_INVENTORY_VERSION,
+        files: [...files.entries()].toSorted(([left], [right]) => left.localeCompare(right)).map(([eval_file, file]) => ({
+            eval_file,
+            tasks: file.tasks.toSorted((left, right) => left.task_key.localeCompare(right.task_key)),
+            ...(file.completeness === 'complete'
+                ? { completeness: 'complete' }
+                : { completeness: 'incomplete', reason: file.reason ?? 'adapter-cannot-prove-completeness' }),
+        })),
+    };
+}
+function taskInventoryEntry(group) {
+    const states = group.cases.map(testCase => testCase.state);
+    if (states.every(state => state === 'skipped')) {
+        return { task_key: group.groupName, state: 'not-scored', reason: 'skipped' };
+    }
+    if (states.every(state => state === 'pending')) {
+        return { task_key: group.groupName, state: 'not-scored', reason: 'pending' };
+    }
+    if (group.cases.some(testCase => testCase.attempts?.some(attempt => attempt.outcome.kind === 'timed-out'))) {
+        return { task_key: group.groupName, state: 'not-scored', reason: 'timed-out' };
+    }
+    if (group.cases.some(isPromotableCase))
+        return { task_key: group.groupName, state: 'scored' };
+    return { task_key: group.groupName, state: 'not-scored', reason: 'failed-before-evaluation' };
+}
+function isPromotableCase(testCase) {
+    if (testCase.state === 'skipped' || testCase.state === 'pending')
+        return false;
+    const evaluations = testCase.attempts?.flatMap(attempt => attempt.evaluations ?? []) ?? testCase.evaluations;
+    if (evaluations === undefined)
+        return true;
+    return evaluations.some(evaluation => evaluation.score !== undefined
+        && evaluation.resultKind !== 'synthetic_no_evaluation');
+}
 function totalDurationMs(runCase) {
     return runCase.attempts.reduce((sum, attempt) => sum + (attempt.durationMs ?? 0), 0);
 }

package/dist/runners/vitest-adapter.js

@@ -31,11 +31,15 @@ export function createVitestAdapter(options = {}) {
                 native: {
                     testModules: options.testModules ?? [],
                     cwd: discoveryCwd,
+                    discoveredFiles: input.discovered.units.flatMap(unit => unit.sourceRef ? [unit.sourceRef] : []),
                 },
             };
         },
         async collectNormalizedRunSnapshot(run) {
-            return buildNormalizedRunSnapshotFromReportGroups(run, collectVitestReportGroups(readVitestRunNative(run).testModules), { cwd: readVitestRunNative(run).cwd });
+            return buildNormalizedRunSnapshotFromReportGroups(run, collectVitestReportGroups(readVitestRunNative(run).testModules), {
+                cwd: readVitestRunNative(run).cwd,
+                discoveredFiles: readVitestRunNative(run).discoveredFiles,
+            });
         },
     };
 }
@@ -59,7 +63,7 @@ export function collectVitestReportGroups(testModules) {
 function readVitestRunNative(run) {
     if (isVitestRunNative(run.native))
         return run.native;
-    return { testModules: [], cwd: process.cwd() };
+    return { testModules: [], cwd: process.cwd(), discoveredFiles: [] };
 }
 function isVitestRunNative(native) {
     return typeof native === 'object'

package/dist/sdk/agent.js

@@ -20,6 +20,8 @@ import { collectOpenCodeMcpToolNames } from '../agents/opencode/contract.js';
 import { collectSensitiveEnvValues } from '../tool-event-results.js';
 import { createAskUserHandler } from './ask-bus/handler.js';
 import { resolveAgentRuntimeOptions } from './agent-runtime-options.js';
+import { extractToolEventsFromLog } from '../tool-events.js';
+import { clearToolEventRuntimeMetadata } from './tool-event-secrets.js';
 /**
  * Test-only injection point: override the sink used by the next emitter
  * built inside `createAgent`. Pass `null` to restore the default (stderr).
@@ -392,7 +394,14 @@ class AgentImpl {
             }
         }
         finally {
-            await this.ws.dispose();
+            try {
+                await this.ws.dispose();
+            }
+            finally {
+                for (const event of extractToolEventsFromLog(this._log)) {
+                    clearToolEventRuntimeMetadata(event);
+                }
+            }
         }
     }
 }

package/dist/sdk/evaluate.js

@@ -7,6 +7,9 @@ import { runScorer } from './run-scorer.js';
 import { createLLMClient } from '../utils/llm.js';
 import { sandboxExec } from '../providers/sandbox-exec.js';
 import { buildTranscript, loadRunSnapshot, WorkspaceMissingError } from './snapshots.js';
+import { sanitizePersistenceValue } from '../tool-event-results.js';
+import { getOriginalMcpInput } from './mcp-event-input.js';
+import { cloneToolEventWithRuntimeMetadata, collectToolEventSensitiveValues, } from './tool-event-secrets.js';
 import fs from 'fs-extra';
 import path from 'path';
 import { summarizeFlow } from './agent-flow.js';
@@ -154,6 +157,9 @@ export function evaluateStepScorers(agent, scorers, opts) {
     return evaluateAgent(agent, scorers, opts, false);
 }
 async function fromSnapshot(snapshotPath, scorers, opts) {
+    if (opts?.deterministicToolEvidence === 'live') {
+        throw new TypeError("evaluate.fromSnapshot() does not support deterministicToolEvidence: 'live'");
+    }
     const snapshot = await loadRunSnapshot(snapshotPath);
     const trackedLLM = opts?.llm ?? createLLMClient({ adapters: [{
                 name: 'runtime', isAvailable: async () => true,
@@ -201,6 +207,9 @@ async function evaluateWithContext(ctx, scorers, opts) {
                 call: (prompt, callOpts) => getRuntime().llm.call(prompt, callOpts),
             }] });
     const onScorerError = opts?.onScorerError ?? 'skip';
+    const deterministicCtx = opts?.deterministicToolEvidence === 'live'
+        ? createLiveDeterministicContext(ctx)
+        : ctx;
     const phase1 = [];
     const phase2 = [];
     const phase3 = [];
@@ -219,7 +228,7 @@ async function evaluateWithContext(ctx, scorers, opts) {
         }
     }
     const results = [];
-    const phase1Results = await Promise.all(phase1.map((g) => runScorer(g, ctx)));
+    const phase1Results = await Promise.all(phase1.map((g) => runScorer(g, deterministicCtx)));
     results.push(...phase1Results);
     const anyCheckFailed = failFast && phase1Results.some((r) => r.type === 'check' && r.status !== 'error' && r.score === 0);
     if (anyCheckFailed) {
@@ -230,7 +239,7 @@ async function evaluateWithContext(ctx, scorers, opts) {
     else {
         const judgeResults = await runJudgePipeline(phase2, ctx, { llm: trackedLLM });
         results.push(...judgeResults);
-        const phase3Results = await Promise.all(phase3.map((g) => runScorer(g, ctx)));
+        const phase3Results = await Promise.all(phase3.map((g) => runScorer(g, deterministicCtx)));
         results.push(...phase3Results);
     }
     const tokenUsage = trackedLLM.tokenUsage ?? { inputTokens: 0, outputTokens: 0 };
@@ -240,12 +249,58 @@ async function evaluateWithContext(ctx, scorers, opts) {
     const totalWeight = scoringResults.reduce((sum, r) => sum + r.weight, 0);
     const weightedSum = scoringResults.reduce((sum, r) => sum + r.score * r.weight, 0);
     const score = totalWeight > 0 ? weightedSum / totalWeight : 0;
-    return {
+    return sanitizePersistenceValue({
         score,
         scorers: results,
         tokenUsage,
+    }, collectToolEventSensitiveValues(ctx.toolEvents));
+}
+function createLiveDeterministicContext(ctx) {
+    return {
+        ...ctx,
+        toolEvents: ctx.toolEvents.map((event) => {
+            const originalInput = event.action === 'mcp_tool_call'
+                ? getOriginalMcpInput(event)
+                : undefined;
+            const mcp = normalizedMcpClassification(event);
+            return originalInput
+                ? cloneToolEventWithRuntimeMetadata(event, { arguments: {
+                        ...event.arguments,
+                        server: event.arguments?.server,
+                        tool: event.arguments?.tool,
+                        status: event.arguments?.status,
+                        ...structuredClone(originalInput),
+                    }, ...(mcp ? { mcp } : {}) })
+                : event;
+        }),
     };
 }
+function normalizedMcpClassification(event) {
+    if (event.mcp)
+        return event.mcp;
+    const args = event.arguments ?? {};
+    const separator = event.providerToolName.indexOf('.');
+    const serverName = typeof args.server === 'string' ? args.server
+        : separator > 0 ? event.providerToolName.slice(0, separator) : undefined;
+    const toolName = typeof args.tool === 'string' ? args.tool
+        : separator > 0 && separator < event.providerToolName.length - 1
+            ? event.providerToolName.slice(separator + 1) : undefined;
+    const status = event.status ?? args.status;
+    if (!serverName || !toolName)
+        return undefined;
+    if (status === 'completed')
+        return { serverName, toolName, invocation: 'confirmed', outcome: 'completed' };
+    if (status === 'error' || status === 'failed') {
+        return { serverName, toolName, invocation: 'confirmed', outcome: 'tool_error' };
+    }
+    if (status === 'user_denied' || status === 'policy_denied') {
+        return { serverName, toolName, invocation: 'not_invoked', outcome: status };
+    }
+    if (status === 'protocol_error') {
+        return { serverName, toolName, invocation: 'unknown', outcome: 'protocol_error' };
+    }
+    return undefined;
+}
 function maybeThrowOnScorerErrors(result, mode) {
     if (mode !== 'fail')
         return;

package/dist/sdk/judge-prompt-builder.js

@@ -1,13 +1,15 @@
 import { sanitizePersistenceValue } from '../tool-event-results.js';
-import { getToolEventSensitiveValues } from './tool-event-secrets.js';
+import { collectToolEventSensitiveValues, } from './tool-event-secrets.js';
 export function buildJudgePrompt(scorer, ctx, input) {
     const sections = [];
-    sections.push(`## Session Transcript\n${ctx.transcript}`);
+    const sensitiveValues = collectToolEventSensitiveValues(ctx.toolEvents);
+    sections.push(`## Session Transcript\n${sanitizePersistenceValue(ctx.transcript, sensitiveValues)}`);
     if (scorer.includeToolEvents && ctx.toolEvents.length > 0) {
         sections.push(`## Tool Events\n${formatToolEvents(ctx)}`);
     }
     if (input) {
-        for (const [key, value] of Object.entries(input)) {
+        const sanitizedInput = sanitizePersistenceValue(input, sensitiveValues);
+        for (const [key, value] of Object.entries(sanitizedInput)) {
             const body = typeof value === 'string' ? value : JSON.stringify(value, null, 2);
             sections.push(`## ${key}\n${body}`);
         }
@@ -22,10 +24,10 @@ ${scorer.rubric}
 Respond with ONLY a JSON object: {"score": <number>, "details": "<brief explanation>"}`;
 }
 function formatToolEvents(ctx) {
+    const sensitiveValues = collectToolEventSensitiveValues(ctx.toolEvents);
     return ctx.toolEvents
         .map((event) => {
         const turn = event.turnNumber ? `turn ${event.turnNumber}` : 'instruction';
-        const sensitiveValues = getToolEventSensitiveValues(event);
         const details = [
             event.status ? `  status: ${event.status}` : undefined,
             event.mcp ? `  mcp: ${JSON.stringify(event.mcp)}` : undefined,
@@ -41,7 +43,8 @@ function formatToolEvents(ctx) {
 }
 export function buildBatchedJudgePrompt(judges, ctx, inputs) {
     const sections = [];
-    sections.push(`## Session Transcript\n${ctx.transcript}`);
+    const sensitiveValues = collectToolEventSensitiveValues(ctx.toolEvents);
+    sections.push(`## Session Transcript\n${sanitizePersistenceValue(ctx.transcript, sensitiveValues)}`);
     if (judges.some((j) => j.includeToolEvents) && ctx.toolEvents.length > 0) {
         sections.push(`## Tool Events\n${formatToolEvents(ctx)}`);
     }
@@ -49,7 +52,8 @@ export function buildBatchedJudgePrompt(judges, ctx, inputs) {
         const parts = [`### Rubric ${i + 1}: "${j.name}"\n${j.rubric}`];
         const input = inputs[i];
         if (input) {
-            for (const [key, value] of Object.entries(input)) {
+            const sanitizedInput = sanitizePersistenceValue(input, sensitiveValues);
+            for (const [key, value] of Object.entries(sanitizedInput)) {
                 const body = typeof value === 'string' ? value : JSON.stringify(value, null, 2);
                 parts.push(`#### ${key}\n${body}`);
             }
@@ -77,7 +81,7 @@ export function buildToolUseJudgePrompt(scorer, ctx) {
     ].join('\n');
     const parts = [];
     parts.push('## Session Transcript');
-    parts.push(ctx.transcript);
+    parts.push(sanitizePersistenceValue(ctx.transcript, collectToolEventSensitiveValues(ctx.toolEvents)));
     if (scorer.includeToolEvents && ctx.toolEvents.length > 0) {
         parts.push('## Tool Events');
         parts.push(formatToolEvents(ctx));

package/dist/sdk/mcp-event-input.d.ts

@@ -1,3 +1,4 @@
 import type { ToolEvent } from '../tool-events.js';
 export declare function attachOriginalMcpInput(event: ToolEvent, input: Record<string, unknown>): ToolEvent;
 export declare function getOriginalMcpInput(event: ToolEvent): Record<string, unknown> | undefined;
+export declare function clearOriginalMcpInput(event: ToolEvent): void;

package/dist/sdk/mcp-event-input.js

@@ -6,3 +6,6 @@ export function attachOriginalMcpInput(event, input) {
 export function getOriginalMcpInput(event) {
     return originalInputs.get(event);
 }
+export function clearOriginalMcpInput(event) {
+    originalInputs.delete(event);
+}

package/dist/sdk/mcp-evidence.js

@@ -2,9 +2,12 @@ export function getMcpToolCall(event) {
     if (event.action !== 'mcp_tool_call')
         return undefined;
     const args = event.arguments ?? {};
-    const serverName = typeof args.server === 'string' ? args.server : undefined;
-    const toolName = typeof args.tool === 'string' ? args.tool : undefined;
-    const status = typeof args.status === 'string' ? args.status : undefined;
+    const serverName = event.mcp?.serverName
+        ?? (typeof args.server === 'string' ? args.server : undefined);
+    const toolName = event.mcp?.toolName
+        ?? (typeof args.tool === 'string' ? args.tool : undefined);
+    const argumentStatus = typeof args.status === 'string' ? args.status : undefined;
+    const status = mcpClassificationStatus(event.mcp, argumentStatus) ?? argumentStatus;
     if (!serverName || !toolName || !status)
         return undefined;
     return {
@@ -15,6 +18,16 @@ export function getMcpToolCall(event) {
         event,
     };
 }
+function mcpClassificationStatus(mcp, argumentStatus) {
+    if (!mcp)
+        return undefined;
+    if (mcp.invocation === 'confirmed') {
+        if (mcp.outcome === 'completed')
+            return 'completed';
+        return argumentStatus === 'failed' || argumentStatus === 'error' ? argumentStatus : 'error';
+    }
+    return mcp.outcome;
+}
 export function isMcpToolCall(event, expected = {}) {
     const call = getMcpToolCall(event);
     return call !== undefined && matchesMcpToolCall(call, expected);

package/dist/sdk/mcp-safety.js

@@ -1,4 +1,4 @@
-import { sanitizePersistenceValue } from '../tool-event-results.js';
+import { sanitizeUntrustedPersistenceValue } from '../tool-event-results.js';
 export function decideMcpToolCall(options, request) {
     const runMode = options?.runMode ?? 'mock';
     if (runMode === 'mock')
@@ -31,7 +31,7 @@ export function decideMcpToolCall(options, request) {
     return { action: 'allow' };
 }
 export function redactMcpSecrets(value) {
-    return sanitizePersistenceValue(value);
+    return sanitizeUntrustedPersistenceValue(value);
 }
 function ruleMatches(rule, request) {
     if (rule.serverName !== undefined && rule.serverName !== request.serverName)

package/dist/sdk/scorers.d.ts

@@ -1,15 +1,18 @@
 import type { CheckScorer, CodeJudgeToolName, ScorerContext, JudgeScorer, ScoreScorer, ScoreResult, ToolExpectation, ToolUsageScorer } from './types.js';
+import type { JsonValue } from '../internal/direct-mcp-v2/types.js';
 /**
  * Create a check scorer — a boolean gate that passes (1.0) or fails (0.0).
  */
 export declare function check(name: string, fn: (ctx: ScorerContext) => boolean | Promise<boolean>, opts?: {
     weight?: number;
+    revision?: JsonValue;
 }): CheckScorer;
 /**
  * Create a score scorer — returns a number (0-1) or { score, details } for partial credit.
  */
 export declare function score(name: string, fn: (ctx: ScorerContext) => number | ScoreResult | Promise<number | ScoreResult>, opts?: {
     weight?: number;
+    revision?: JsonValue;
 }): ScoreScorer;
 /**
  * Create a judge scorer — uses an LLM to evaluate against a rubric.
@@ -55,10 +58,12 @@ export declare function judge(name: string, opts: {
     maxRounds?: number;
     /** Anthropic prompt caching. Default: true when tools is set, unchanged otherwise. */
     cacheControl?: boolean;
+    revision?: JsonValue;
 }): JudgeScorer;
 /**
  * Create a tool usage scorer — matches tool events against expectations.
  */
 export declare function toolUsage(name: string, expectations: ToolExpectation[], opts?: {
     weight?: number;
+    revision?: JsonValue;
 }): ToolUsageScorer;

package/dist/sdk/scorers.js

@@ -7,6 +7,7 @@ export function check(name, fn, opts) {
         name,
         weight: opts?.weight ?? 1,
         fn,
+        ...(opts?.revision === undefined ? {} : { revision: opts.revision }),
     };
 }
 /**
@@ -18,6 +19,7 @@ export function score(name, fn, opts) {
         name,
         weight: opts?.weight ?? 1,
         fn,
+        ...(opts?.revision === undefined ? {} : { revision: opts.revision }),
     };
 }
 /**
@@ -65,6 +67,7 @@ export function judge(name, opts) {
         tools,
         maxRounds: opts.maxRounds,
         cacheControl,
+        ...(opts.revision === undefined ? {} : { revision: opts.revision }),
     };
 }
 /**
@@ -76,5 +79,6 @@ export function toolUsage(name, expectations, opts) {
         name,
         weight: opts?.weight ?? 1,
         expectations,
+        ...(opts?.revision === undefined ? {} : { revision: opts.revision }),
     };
 }

package/dist/sdk/scripted-mcp-events.js

@@ -1,4 +1,5 @@
 import { redactMcpSecrets } from './mcp-safety.js';
+import { attachLiveMcpInput } from './tool-event-secrets.js';
 export function buildScriptedMcpApprovalEvent(opts) {
     const providerToolName = `${opts.serverName}.${opts.toolName}`;
     return {
@@ -18,7 +19,7 @@ export function buildScriptedMcpApprovalEvent(opts) {
 export function buildScriptedMcpDeniedCallEvent(opts) {
     const providerToolName = `${opts.serverName}.${opts.toolName}`;
     const args = redactMcpSecrets(opts.args);
-    return {
+    return attachLiveMcpInput({
         action: 'mcp_tool_call', provider: opts.provider, providerToolName,
         toolUseId: opts.toolUseId, turnNumber: opts.turnNumber, status: 'error',
         mcp: { serverName: opts.serverName, toolName: opts.toolName, invocation: 'not_invoked', outcome: opts.outcome },
@@ -26,5 +27,5 @@ export function buildScriptedMcpDeniedCallEvent(opts) {
         summary: `MCP tool ${providerToolName} ${opts.outcome}`,
         confidence: 'high',
         rawSnippet: JSON.stringify({ status: opts.outcome }),
-    };
+    }, opts.args);
 }

package/dist/sdk/tool-event-log.js

@@ -1,7 +1,22 @@
-import { sanitizePersistenceValue } from '../tool-event-results.js';
-import { getToolEventSensitiveValues } from './tool-event-secrets.js';
+import { collectStructuredSensitiveValues, isSensitiveValueScanLimitError, sanitizePersistenceValue, } from '../tool-event-results.js';
+import { attachToolEventSensitiveValues, cloneToolEventWithRuntimeMetadata, getToolEventSensitiveValues, redactToolEventPayload, } from './tool-event-secrets.js';
 export function buildToolEventLogEntry(toolEvent, fallbackTimestamp) {
-    const persistedEvent = sanitizePersistenceValue(toolEvent, getToolEventSensitiveValues(toolEvent));
+    let event = toolEvent;
+    let discoveredValues;
+    try {
+        discoveredValues = collectStructuredSensitiveValues(event);
+    }
+    catch (error) {
+        if (!isSensitiveValueScanLimitError(error))
+            throw error;
+        event = redactToolEventPayload(event);
+        discoveredValues = [];
+    }
+    const sensitiveValues = [...new Set([
+            ...getToolEventSensitiveValues(toolEvent),
+            ...discoveredValues,
+        ])];
+    const persistedEvent = attachToolEventSensitiveValues(cloneToolEventWithRuntimeMetadata(event, sanitizePersistenceValue(event, sensitiveValues)), sensitiveValues);
     return {
         type: 'tool_event',
         timestamp: persistedEvent.startedAt ?? fallbackTimestamp,

package/dist/sdk/tool-event-secrets.d.ts

@@ -1,4 +1,8 @@
 import type { ToolEvent } from '../tool-events.js';
 export declare function attachToolEventSensitiveValues(event: ToolEvent, sensitiveValues: readonly string[]): ToolEvent;
 export declare function getToolEventSensitiveValues(event: ToolEvent): readonly string[];
+export declare function attachLiveMcpInput(event: ToolEvent, input: Record<string, unknown>): ToolEvent;
+export declare function redactToolEventPayload(event: ToolEvent): ToolEvent;
+export declare function collectToolEventSensitiveValues(events: readonly ToolEvent[]): string[];
+export declare function clearToolEventRuntimeMetadata(event: ToolEvent): void;
 export declare function cloneToolEventWithRuntimeMetadata(source: ToolEvent, overrides: Partial<ToolEvent>): ToolEvent;