@wix/pathgrade 1.0.40 → 1.0.42

package/README.md

@@ -321,7 +321,7 @@ Pathgrade exposes a few useful features that are easy to miss from the basic exa
 - `createAgent({ skillDir, workspace })` stages a real skill and a fixture workspace into the sandbox, which is how Pathgrade's skill examples are evaluated.
 - `createAgent({ debug: true })` preserves the final workspace under the backward-compatible `pathgrade-debug/<test-name>/` path; when you use `runConversation()`, it also writes `run-snapshot.json`.
 - `createAgent({ debug: { retainRuns: 5 } })` opts into managed run retention under `pathgrade-debug/runs/<run-id>/<test-name>/`. Use `pathgrade clean --debug`, `--keep=N`, and `--dry-run` to clean marked, inactive debug runs safely.
-- `evaluate.fromSnapshot(snapshotPath, scorers)` re-runs grading against a saved snapshot without re-running the agent.
+- `evaluate.fromSnapshot(snapshotPath, scorers)` re-runs grading against a saved snapshot without re-running the agent. For trusted live deterministic scoring, `evaluate(agent, scorers, { deterministicToolEvidence: 'live' })` exposes exact transient MCP arguments only through `ctx.toolEvents`; `agent.log` stays sanitized, and redaction-blocked argument predicates fail closed.
 - `previewReactions(messages, reactions)` lets you inspect which scripted reactions would fire offline.
 - `conversationWindow` on agents and personas keeps long transcripts bounded with summarization instead of sending the full conversation every turn.
 - `copyIgnore` and `DEFAULT_COPY_IGNORE` let you control what gets copied into the sandbox when seeding from large fixtures or skill directories.

package/dist/pathgrade.d.ts

@@ -10,5 +10,6 @@
 export interface PathgradeCliOptions {
     name?: string;
     version?: string;
+    blockedEnvironmentKeys?: readonly string[];
 }
 export declare function runPathgradeCli(options?: PathgradeCliOptions): Promise<void>;

package/dist/pathgrade.js

@@ -182,6 +182,8 @@ export async function runPathgradeCli(options = {}) {
     if (command === 'run' || !command || command.startsWith('-')) {
         // pathgrade run [--changed [--since=…|--changed-files=…]] [--] [runner-args]
         loadDotenv();
+        for (const key of options.blockedEnvironmentKeys ?? [])
+            delete process.env[key];
         validateApiKeys();
         const parsed = parsePathgradeRunArgs(command === 'run' ? args.slice(1) : args);
         for (const warning of parsed.warnings ?? []) {

package/dist/sdk/evaluate.js

@@ -270,7 +270,7 @@ function createLiveDeterministicContext(ctx) {
                         tool: event.arguments?.tool,
                         status: event.arguments?.status,
                         ...structuredClone(originalInput),
-                    }, ...(mcp ? { mcp } : {}) })
+                    }, redactedArgumentPaths: undefined, ...(mcp ? { mcp } : {}) })
                 : event;
         }),
     };

package/dist/sdk/index.d.ts

@@ -4,7 +4,7 @@ export { AgentCrashError } from './agent-crash.js';
 export { check, score, judge, toolUsage } from './scorers.js';
 export { compileScenario, compileScenarioText } from './scenario-machine-v2.js';
 export { findScenarioCaseMatches, getFinalScenarioState, getScenarioStateTimeline, ScenarioEvidenceError, wasScenarioCaseMatched, } from './scenario-evidence.js';
-export { getMcpToolCall, isMcpToolCall, findMcpToolCalls, getMcpStartupStatus, isMcpStartupStatus, getMcpApproval, isMcpApproval, findMcpApprovals, getMcpInvocation, wasMcpToolInvoked, } from './mcp-evidence.js';
+export { getMcpToolCall, isMcpToolCall, findMcpToolCalls, McpEvidenceUnavailableError, getMcpStartupStatus, isMcpStartupStatus, getMcpApproval, isMcpApproval, findMcpApprovals, getMcpInvocation, wasMcpToolInvoked, } from './mcp-evidence.js';
 export type { McpMockApprovalRule, McpMockJsonValue } from './mcp-mock-approvals.js';
 export type { CompileScenarioResult, EvidenceEnvelope, JsonObject, JsonPrimitive, JsonValue, MatchExpr, MatchPredicate, McpContent, ScenarioArtifact, ScenarioCase, ScenarioDiagnostic, ScenarioEvidence, ScenarioMachineV2, ScenarioRejection, ScenarioSuccess, ScenarioToolError, } from './scenario-machine-v2.js';
 export type { ExpectedScenarioCaseMatch, ScenarioCaseMatchEvidence, ScenarioStateTransitionEvidence, } from './scenario-evidence.js';

package/dist/sdk/index.js

@@ -5,7 +5,7 @@ export { AgentCrashError } from './agent-crash.js';
 export { check, score, judge, toolUsage } from './scorers.js';
 export { compileScenario, compileScenarioText } from './scenario-machine-v2.js';
 export { findScenarioCaseMatches, getFinalScenarioState, getScenarioStateTimeline, ScenarioEvidenceError, wasScenarioCaseMatched, } from './scenario-evidence.js';
-export { getMcpToolCall, isMcpToolCall, findMcpToolCalls, getMcpStartupStatus, isMcpStartupStatus, getMcpApproval, isMcpApproval, findMcpApprovals, getMcpInvocation, wasMcpToolInvoked, } from './mcp-evidence.js';
+export { getMcpToolCall, isMcpToolCall, findMcpToolCalls, McpEvidenceUnavailableError, getMcpStartupStatus, isMcpStartupStatus, getMcpApproval, isMcpApproval, findMcpApprovals, getMcpInvocation, wasMcpToolInvoked, } from './mcp-evidence.js';
 export { decideMcpToolCall, redactMcpSecrets, } from './mcp-safety.js';
 export { evaluate, evaluateFlow, EvalScorerError } from './evaluate.js';
 export { RUN_SNAPSHOT_VERSION, buildRunSnapshot, loadRunSnapshot, SnapshotParseError, SnapshotVersionError, WorkspaceMissingError, } from './snapshots.js';

package/dist/sdk/mcp-evidence.d.ts

@@ -1,4 +1,7 @@
 import type { ToolEvent } from '../tool-events.js';
+export declare class McpEvidenceUnavailableError extends TypeError {
+    readonly name = "McpEvidenceUnavailableError";
+}
 export type McpInvocationResult = {
     serverName: string;
     toolName: string;
@@ -46,6 +49,7 @@ export interface ExpectedMcpToolCall {
     status?: string;
     invocation?: McpInvocationResult['invocation'];
     outcome?: McpInvocationResult['outcome'];
+    /** Throws when persisted redaction makes the requested argument match unknowable. */
     argumentsContaining?: Record<string, unknown>;
 }
 export interface McpStartupStatusEvidence {

package/dist/sdk/mcp-evidence.js

@@ -1,13 +1,21 @@
+const REDACTED_ARGUMENTS_ERROR = "argumentsContaining cannot be evaluated against redacted MCP arguments; use the scorer's ctx.toolEvents with deterministicToolEvidence: 'live'";
+export class McpEvidenceUnavailableError extends TypeError {
+    name = 'McpEvidenceUnavailableError';
+}
 export function getMcpToolCall(event) {
     if (event.action !== 'mcp_tool_call')
         return undefined;
     const args = event.arguments ?? {};
+    const separator = event.providerToolName.indexOf('.');
     const serverName = event.mcp?.serverName
-        ?? (typeof args.server === 'string' ? args.server : undefined);
+        ?? (typeof args.server === 'string' ? args.server : undefined)
+        ?? (separator > 0 ? event.providerToolName.slice(0, separator) : undefined);
     const toolName = event.mcp?.toolName
-        ?? (typeof args.tool === 'string' ? args.tool : undefined);
+        ?? (typeof args.tool === 'string' ? args.tool : undefined)
+        ?? (separator > 0 && separator < event.providerToolName.length - 1
+            ? event.providerToolName.slice(separator + 1) : undefined);
     const argumentStatus = typeof args.status === 'string' ? args.status : undefined;
-    const status = mcpClassificationStatus(event.mcp, argumentStatus) ?? argumentStatus;
+    const status = mcpClassificationStatus(event.mcp, argumentStatus) ?? argumentStatus ?? event.status;
     if (!serverName || !toolName || !status)
         return undefined;
     return {
@@ -151,29 +159,46 @@ function matchesMcpToolCall(call, expected) {
         if (expected.outcome !== undefined && invocation.outcome !== expected.outcome)
             return false;
     }
-    if (expected.argumentsContaining && !containsArguments(call.arguments, expected.argumentsContaining)) {
+    if (expected.argumentsContaining && !containsArguments(call.arguments, expected.argumentsContaining, call.event.redactedArgumentPaths)) {
         return false;
     }
     return true;
 }
-function containsArguments(actual, expected) {
-    return Object.entries(expected).every(([key, expectedValue]) => valuesEqual(actual[key], expectedValue));
+function containsArguments(actual, expected, redactedPaths = []) {
+    const matches = Object.entries(expected).every(([key, expectedValue]) => valuesEqual(actual[key], expectedValue, `/${escapeJsonPointer(key)}`, redactedPaths));
+    return matches;
 }
-function valuesEqual(actual, expected) {
-    if (Object.is(actual, expected))
+function valuesEqual(actual, expected, path, redactedPaths) {
+    if (Object.is(actual, expected)) {
+        assertMcpArgumentAvailable(path, redactedPaths);
         return true;
+    }
     if (Array.isArray(actual) || Array.isArray(expected)) {
-        if (!Array.isArray(actual) || !Array.isArray(expected))
-            return false;
-        if (actual.length !== expected.length)
-            return false;
-        return actual.every((value, index) => valuesEqual(value, expected[index]));
+        if (Array.isArray(actual) && Array.isArray(expected) && actual.length === expected.length) {
+            return actual.every((value, index) => valuesEqual(value, expected[index], `${path}/${index}`, redactedPaths));
+        }
+        return rejectRedactedMismatch(path, redactedPaths);
     }
     if (isRecord(actual) && isRecord(expected)) {
-        return Object.entries(expected).every(([key, value]) => valuesEqual(actual[key], value));
+        return Object.entries(expected).every(([key, value]) => valuesEqual(actual[key], value, `${path}/${escapeJsonPointer(key)}`, redactedPaths));
     }
+    return rejectRedactedMismatch(path, redactedPaths);
+}
+function rejectRedactedMismatch(path, redactedPaths) {
+    assertMcpArgumentAvailable(path, redactedPaths);
     return false;
 }
+function assertMcpArgumentAvailable(path, redactedPaths) {
+    if (redactedPaths.some((redactedPath) => redactedPath === ''
+        || redactedPath === path
+        || redactedPath.startsWith(`${path}/`)
+        || path.startsWith(`${redactedPath}/`))) {
+        throw new McpEvidenceUnavailableError(REDACTED_ARGUMENTS_ERROR);
+    }
+}
+function escapeJsonPointer(value) {
+    return value.replaceAll('~', '~0').replaceAll('/', '~1');
+}
 function isRecord(value) {
     return !!value && typeof value === 'object' && !Array.isArray(value);
 }

package/dist/sdk/run-scorer.js

@@ -1,6 +1,7 @@
 import { runJudgePipeline } from './judge-pipeline.js';
 import { runJudgeSession } from './judge-tool-session.js';
 import { clamp, makeErroredResult, matchesExpectation } from './scorer-utils.js';
+import { McpEvidenceUnavailableError } from './mcp-evidence.js';
 export async function runScorer(scorer, ctx, opts = {}) {
     switch (scorer.type) {
         case 'check':
@@ -56,6 +57,9 @@ async function runCheckScorer(scorer, ctx) {
         };
     }
     catch (error) {
+        if (error instanceof McpEvidenceUnavailableError) {
+            return makeUnavailableEvidenceResult('check', scorer.name, scorer.weight, error);
+        }
         return makeErroredResult('check', scorer.name, scorer.weight, error);
     }
 }
@@ -81,9 +85,15 @@ async function runScoreScorer(scorer, ctx) {
         };
     }
     catch (error) {
+        if (error instanceof McpEvidenceUnavailableError) {
+            return makeUnavailableEvidenceResult('score', scorer.name, scorer.weight, error);
+        }
         return makeErroredResult('score', scorer.name, scorer.weight, error);
     }
 }
+function makeUnavailableEvidenceResult(type, name, weight, error) {
+    return { name, type, score: 0, weight, details: error.message, status: 'ok' };
+}
 async function runToolUsageScorer(scorer, ctx) {
     try {
         const toolEvents = ctx.toolEvents;

package/dist/sdk/tool-event-log.js

@@ -1,5 +1,8 @@
+import { isDeepStrictEqual } from 'node:util';
 import { collectStructuredSensitiveValues, isSensitiveValueScanLimitError, sanitizePersistenceValue, } from '../tool-event-results.js';
 import { attachToolEventSensitiveValues, cloneToolEventWithRuntimeMetadata, getToolEventSensitiveValues, redactToolEventPayload, } from './tool-event-secrets.js';
+import { getOriginalMcpInput } from './mcp-event-input.js';
+const MCP_METADATA_ARGUMENTS = new Set(['server', 'tool', 'status']);
 export function buildToolEventLogEntry(toolEvent, fallbackTimestamp) {
     let event = toolEvent;
     let discoveredValues;
@@ -16,10 +19,37 @@ export function buildToolEventLogEntry(toolEvent, fallbackTimestamp) {
             ...getToolEventSensitiveValues(toolEvent),
             ...discoveredValues,
         ])];
-    const persistedEvent = attachToolEventSensitiveValues(cloneToolEventWithRuntimeMetadata(event, sanitizePersistenceValue(event, sensitiveValues)), sensitiveValues);
+    const sanitizedEvent = sanitizePersistenceValue(event, sensitiveValues);
+    const originalInput = getOriginalMcpInput(event);
+    const originalArguments = originalInput ? {
+        ...event.arguments,
+        ...Object.fromEntries(Object.entries(originalInput).filter(([key]) => !MCP_METADATA_ARGUMENTS.has(key))),
+    } : event.arguments;
+    const redactedArgumentPaths = [...new Set([
+            ...(event.redactedArgumentPaths ?? []),
+            ...collectChangedPaths(originalArguments, sanitizedEvent.arguments),
+        ])];
+    const persistedEvent = attachToolEventSensitiveValues(cloneToolEventWithRuntimeMetadata(event, {
+        ...sanitizedEvent,
+        ...(redactedArgumentPaths.length > 0 ? { redactedArgumentPaths } : {}),
+    }), sensitiveValues);
     return {
         type: 'tool_event',
         timestamp: persistedEvent.startedAt ?? fallbackTimestamp,
         tool_event: persistedEvent,
     };
 }
+function collectChangedPaths(original, sanitized, path = '') {
+    if (isDeepStrictEqual(original, sanitized))
+        return [];
+    if (Array.isArray(original) && Array.isArray(sanitized) && original.length === sanitized.length) {
+        return original.flatMap((value, index) => collectChangedPaths(value, sanitized[index], `${path}/${index}`));
+    }
+    if (isRecord(original) && isRecord(sanitized)) {
+        return Object.keys(original).flatMap((key) => collectChangedPaths(original[key], sanitized[key], `${path}/${key.replaceAll('~', '~0').replaceAll('/', '~1')}`));
+    }
+    return [path];
+}
+function isRecord(value) {
+    return !!value && typeof value === 'object' && !Array.isArray(value);
+}

package/dist/sdk/tool-event-secrets.js

@@ -27,6 +27,7 @@ export function redactToolEventPayload(event) {
     return {
         ...event,
         arguments: { redacted: true },
+        redactedArgumentPaths: [''],
         summary: `${event.action} via ${event.providerToolName}`,
         rawSnippet: '<redacted>',
     };

package/dist/sdk/types.d.ts

@@ -348,6 +348,7 @@ export interface ScorerContext {
     workspace: string;
     log: LogEntry[];
     transcript: string;
+    /** Use this scorer-context field for live MCP arguments; `log` always remains sanitized. */
     toolEvents: import('../tool-events.js').ToolEvent[];
     runCommand: (cmd: string) => Promise<CommandResult>;
     artifacts: SessionArtifacts;
@@ -360,7 +361,7 @@ export interface EvaluateOptions {
     failFast?: boolean;
     llm?: LLMPort;
     onScorerError?: 'skip' | 'zero' | 'fail';
-    /** Evidence visible to deterministic scorers. Live mode is unavailable for snapshot replay. */
+    /** Restores exact MCP arguments only on deterministic scorers' `ctx.toolEvents`; `agent.log` remains sanitized. Live mode is unavailable for snapshot replay. */
     deterministicToolEvidence?: 'persisted' | 'live';
     /** Stable identity for this evaluation definition across separate runs. */
     evaluationDefinitionKey?: string;

package/dist/tool-events.d.ts

@@ -20,6 +20,8 @@ export interface ToolEvent {
     toolUseId?: string;
     turnNumber?: number;
     arguments?: Record<string, unknown>;
+    /** Producer-controlled JSON Pointer paths identifying persisted arguments that were sanitized. */
+    redactedArgumentPaths?: string[];
     /** Lifecycle state observed by PathGrade. Absent on legacy provider events. */
     status?: 'completed' | 'error' | 'incomplete';
     /** Receipt/enforcement-backed classification for canonical MCP calls. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wix/pathgrade",
-  "version": "1.0.40",
+  "version": "1.0.42",
   "packageManager": "yarn@4.12.0",
   "description": "Evaluate whether AI agents discover and use your skills correctly",
   "exports": {
@@ -142,5 +142,5 @@
     "typescript": "^5.9.3",
     "zod": "4.3.6"
   },
-  "falconPackageHash": "4c4d43a7c8d439f02756404accbc72b44db02c52db4d0c2b2d18452f"
+  "falconPackageHash": "9b508a508d9397bfc0a4ac6acd286c9019230468e7fad14edcb26c2c"
 }