@wix/pathgrade 1.0.13 → 1.0.15

package/dist/sdk/mcp-mock-approvals.js

@@ -0,0 +1,235 @@
+import { createHmac, randomBytes } from 'node:crypto';
+import { assertScriptedMcpSchemaProfile } from '../core/mcp-schema-profile.js';
+import { canonicalizeJson } from '../core/canonical-json.js';
+const RULE_KEYS = new Set(['serverName', 'toolName', 'argumentsContaining', 'decision', 'label']);
+const INSTRUCTIONS_MAX_BYTES = 32 * 1024;
+const WHEN_MAX_BYTES = 8 * 1024;
+function isPlainObject(value) {
+    if (value === null || typeof value !== 'object' || Array.isArray(value))
+        return false;
+    const prototype = Object.getPrototypeOf(value);
+    return prototype === Object.prototype || prototype === null;
+}
+function cloneJson(value, path, ancestors = new Set()) {
+    if (value === null || typeof value === 'string' || typeof value === 'boolean')
+        return value;
+    if (typeof value === 'number') {
+        if (!Number.isFinite(value))
+            throw new Error(`${path} must contain only finite JSON numbers`);
+        return value;
+    }
+    if (Array.isArray(value)) {
+        if (ancestors.has(value))
+            throw new Error(`${path} must be acyclic JSON`);
+        if (Object.keys(value).some((key) => !/^(0|[1-9]\d*)$/.test(key) || Number(key) >= value.length)) {
+            throw new Error(`${path} must not contain non-index array properties`);
+        }
+        ancestors.add(value);
+        const cloned = Array.from({ length: value.length }, (_, index) => {
+            if (!Object.prototype.hasOwnProperty.call(value, index))
+                throw new Error(`${path}[${index}] must not be sparse`);
+            return cloneJson(value[index], `${path}[${index}]`, ancestors);
+        });
+        ancestors.delete(value);
+        return cloned;
+    }
+    if (!isPlainObject(value))
+        throw new Error(`${path} must contain only plain JSON values`);
+    if (ancestors.has(value))
+        throw new Error(`${path} must be acyclic JSON`);
+    ancestors.add(value);
+    const cloned = {};
+    for (const key of Reflect.ownKeys(value)) {
+        if (typeof key !== 'string')
+            throw new Error(`${path} must not contain symbol keys`);
+        const descriptor = Object.getOwnPropertyDescriptor(value, key);
+        if (!descriptor.enumerable || !('value' in descriptor)) {
+            throw new Error(`${path}.${key} must be an enumerable data property`);
+        }
+        cloned[key] = cloneJson(descriptor.value, `${path}.${key}`, ancestors);
+    }
+    ancestors.delete(value);
+    return cloned;
+}
+function deepFreeze(value, seen = new Set()) {
+    if (value === null || typeof value !== 'object' || seen.has(value))
+        return value;
+    seen.add(value);
+    if (value instanceof Map) {
+        for (const [key, entry] of value) {
+            deepFreeze(key, seen);
+            deepFreeze(entry, seen);
+        }
+        if (!Object.prototype.hasOwnProperty.call(value, 'set')) {
+            const immutable = () => { throw new TypeError('Compiled MCP manifest is immutable'); };
+            Object.defineProperties(value, {
+                set: { value: immutable },
+                delete: { value: immutable },
+                clear: { value: immutable },
+            });
+        }
+    }
+    else {
+        for (const entry of Object.values(value))
+            deepFreeze(entry, seen);
+    }
+    return Object.freeze(value);
+}
+function nonEmptyString(value, path) {
+    if (typeof value !== 'string' || value.length === 0)
+        throw new Error(`${path} must be a non-empty string`);
+}
+function cloneDescriptor(value, index) {
+    const cloned = cloneJson(value, `mcpMock[${index}]`);
+    if (cloned.__type !== 'mock_mcp_server' || !isPlainObject(cloned.config)) {
+        throw new Error(`mcpMock[${index}] must be a generated mock descriptor`);
+    }
+    nonEmptyString(cloned.config.name, `mcpMock[${index}].config.name`);
+    if (!Array.isArray(cloned.config.tools) || cloned.config.tools.length === 0) {
+        throw new Error(`mcpMock[${index}].config.tools must be a non-empty array`);
+    }
+    if (cloned.config.instructions !== undefined) {
+        nonEmptyString(cloned.config.instructions, `mcpMock[${index}].config.instructions`);
+        if (Buffer.byteLength(cloned.config.instructions, 'utf8') > INSTRUCTIONS_MAX_BYTES) {
+            throw new Error(`mcpMock[${index}].config.instructions exceeds 32 KiB`);
+        }
+    }
+    return cloned;
+}
+function buildServer(descriptor) {
+    const byName = new Map();
+    for (const [index, tool] of descriptor.config.tools.entries()) {
+        nonEmptyString(tool.name, `${descriptor.config.name}.tools[${index}].name`);
+        if (tool.when !== undefined) {
+            nonEmptyString(tool.when, `${descriptor.config.name}.${tool.name}.when`);
+            if (Buffer.byteLength(tool.when, 'utf8') > WHEN_MAX_BYTES) {
+                throw new Error(`${descriptor.config.name}.${tool.name}.when exceeds 8 KiB`);
+            }
+            try {
+                new RegExp(tool.when, 'i');
+            }
+            catch {
+                throw new Error(`${descriptor.config.name}.${tool.name}.when is invalid`);
+            }
+        }
+        const schema = tool.inputSchema ?? { type: 'object' };
+        if (!isPlainObject(schema))
+            throw new Error(`${descriptor.config.name}.${tool.name}.inputSchema must be an object`);
+        assertScriptedMcpSchemaProfile(schema, { serverName: descriptor.config.name, toolName: tool.name });
+        const cases = byName.get(tool.name) ?? [];
+        cases.push(tool);
+        byName.set(tool.name, cases);
+    }
+    const tools = new Map();
+    for (const [name, cases] of byName) {
+        const hints = new Set(cases.flatMap((entry) => entry.annotations?.readOnlyHint === undefined
+            ? []
+            : [entry.annotations.readOnlyHint]));
+        if (hints.size > 1)
+            throw new Error(`Conflicting readOnlyHint values for tool "${name}"`);
+        const first = cases[0];
+        tools.set(name, deepFreeze({
+            name,
+            ...(first.description !== undefined ? { description: first.description } : {}),
+            inputSchema: first.inputSchema ?? { type: 'object' },
+            ...(hints.size === 1 ? { readOnlyHint: [...hints][0] } : {}),
+            cases: cases.map((entry) => ({
+                ...(entry.when !== undefined ? { when: entry.when } : {}),
+                response: entry.response,
+            })),
+        }));
+    }
+    return deepFreeze({
+        name: descriptor.config.name,
+        ...(descriptor.config.instructions !== undefined ? { instructions: descriptor.config.instructions } : {}),
+        tools,
+    });
+}
+function selectorSubsumes(earlier, later) {
+    if (Object.is(earlier, later))
+        return true;
+    if (Array.isArray(earlier) || Array.isArray(later)) {
+        return Array.isArray(earlier) && Array.isArray(later)
+            && earlier.length === later.length
+            && earlier.every((entry, index) => selectorSubsumes(entry, later[index]));
+    }
+    if (!isPlainObject(earlier) || !isPlainObject(later))
+        return false;
+    return Object.entries(earlier).every(([key, value]) => Object.prototype.hasOwnProperty.call(later, key) && selectorSubsumes(value, later[key]));
+}
+function assertClaudeNamesUnambiguous(names) {
+    for (const name of names) {
+        if (name.includes('__'))
+            throw new Error(`Claude MCP server name "${name}" contains reserved delimiter "__"`);
+    }
+    for (let left = 0; left < names.length; left++) {
+        for (let right = left + 1; right < names.length; right++) {
+            if (names[left].startsWith(`${names[right]}__`) || names[right].startsWith(`${names[left]}__`)) {
+                throw new Error(`Claude MCP server names "${names[left]}" and "${names[right]}" are ambiguous`);
+            }
+        }
+    }
+}
+export function compileMcpMockApprovalSession(opts) {
+    if (!Array.isArray(opts.rules) || opts.rules.length === 0) {
+        throw new Error('mcpMockApprovalRules must be a non-empty array');
+    }
+    const descriptors = (Array.isArray(opts.mcpMock) ? opts.mcpMock : [opts.mcpMock])
+        .map((descriptor, index) => cloneDescriptor(descriptor, index));
+    const servers = new Map();
+    for (const descriptor of descriptors) {
+        if (servers.has(descriptor.config.name))
+            throw new Error(`Duplicate mock MCP server name: "${descriptor.config.name}"`);
+        servers.set(descriptor.config.name, buildServer(descriptor));
+    }
+    if (opts.provider === 'claude')
+        assertClaudeNamesUnambiguous([...servers.keys()]);
+    const rules = [];
+    for (const [index, source] of opts.rules.entries()) {
+        if (!isPlainObject(source))
+            throw new Error(`mcpMockApprovalRules[${index}] must be a plain object`);
+        for (const key of Object.keys(source)) {
+            if (!RULE_KEYS.has(key))
+                throw new Error(`mcpMockApprovalRules[${index}] has unknown key "${key}"`);
+        }
+        const rule = cloneJson(source, `mcpMockApprovalRules[${index}]`);
+        nonEmptyString(rule.serverName, `mcpMockApprovalRules[${index}].serverName`);
+        nonEmptyString(rule.toolName, `mcpMockApprovalRules[${index}].toolName`);
+        if (rule.decision !== 'approve' && rule.decision !== 'deny') {
+            throw new Error(`mcpMockApprovalRules[${index}].decision must be approve or deny`);
+        }
+        if (rule.label !== undefined)
+            nonEmptyString(rule.label, `mcpMockApprovalRules[${index}].label`);
+        if (rule.argumentsContaining !== undefined && !isPlainObject(rule.argumentsContaining)) {
+            throw new Error(`mcpMockApprovalRules[${index}].argumentsContaining must be a plain JSON object`);
+        }
+        const tool = servers.get(rule.serverName)?.tools.get(rule.toolName);
+        if (!tool)
+            throw new Error(`mcpMockApprovalRules[${index}] targets unknown generated tool ${rule.serverName}.${rule.toolName}`);
+        if (tool.readOnlyHint === true)
+            throw new Error(`mcpMockApprovalRules[${index}] targets read-only tool ${rule.serverName}.${rule.toolName}`);
+        for (let previous = 0; previous < index; previous++) {
+            const earlier = rules[previous];
+            if (earlier.serverName === rule.serverName && earlier.toolName === rule.toolName
+                && (earlier.argumentsContaining === undefined
+                    || (rule.argumentsContaining !== undefined
+                        && selectorSubsumes(earlier.argumentsContaining, rule.argumentsContaining)))) {
+                throw new Error(`mcpMockApprovalRules[${index}] is shadowed by rule ${previous}`);
+            }
+        }
+        rules.push(deepFreeze(rule));
+    }
+    return deepFreeze({ servers, rules });
+}
+export function matchesMcpApprovalArguments(actual, expected) {
+    return expected === undefined || selectorSubsumes(expected, actual);
+}
+export function canonicalizeMcpArguments(value) {
+    return canonicalizeJson(value);
+}
+export function createMcpArgumentDigest(key, value) {
+    return createHmac('sha256', key).update(canonicalizeMcpArguments(value)).digest('base64url');
+}
+export function createMcpReceiptKey() {
+    return randomBytes(32);
+}

package/dist/sdk/scripted-mcp-events.d.ts

@@ -0,0 +1,24 @@
+import type { ToolEvent } from '../tool-events.js';
+interface ScriptedMcpDecisionEvidence {
+    decision: 'approve' | 'deny';
+    outcome: 'matched' | 'unmatched' | 'protocol_error';
+    reason: string;
+    ruleIndex?: number;
+    ruleLabel?: string;
+}
+interface ScriptedMcpEventIdentity {
+    provider: 'claude' | 'codex';
+    serverName: string;
+    toolName: string;
+    toolUseId: string;
+    turnNumber: number;
+}
+export declare function buildScriptedMcpApprovalEvent(opts: ScriptedMcpEventIdentity & {
+    decision: ScriptedMcpDecisionEvidence;
+    status?: ToolEvent['status'];
+}): ToolEvent;
+export declare function buildScriptedMcpDeniedCallEvent(opts: ScriptedMcpEventIdentity & {
+    args: Record<string, unknown>;
+    outcome: 'user_denied' | 'protocol_error';
+}): ToolEvent;
+export {};

package/dist/sdk/scripted-mcp-events.js

@@ -0,0 +1,30 @@
+import { redactMcpSecrets } from './mcp-safety.js';
+export function buildScriptedMcpApprovalEvent(opts) {
+    const providerToolName = `${opts.serverName}.${opts.toolName}`;
+    return {
+        action: 'mcp_approval', provider: opts.provider, providerToolName,
+        toolUseId: opts.toolUseId, turnNumber: opts.turnNumber, status: opts.status ?? 'completed',
+        arguments: {
+            server: opts.serverName, tool: opts.toolName, decision: opts.decision.decision,
+            outcome: opts.decision.outcome, reason: opts.decision.reason, decisionSource: 'scripted_user',
+            ...(opts.decision.ruleIndex !== undefined ? { ruleIndex: opts.decision.ruleIndex } : {}),
+            ...(opts.decision.ruleLabel ? { ruleLabel: opts.decision.ruleLabel } : {}),
+        },
+        summary: `MCP approval ${providerToolName} ${opts.decision.decision} ${opts.decision.outcome}`,
+        confidence: 'high',
+        rawSnippet: JSON.stringify({ decision: opts.decision.decision, outcome: opts.decision.outcome }),
+    };
+}
+export function buildScriptedMcpDeniedCallEvent(opts) {
+    const providerToolName = `${opts.serverName}.${opts.toolName}`;
+    const args = redactMcpSecrets(opts.args);
+    return {
+        action: 'mcp_tool_call', provider: opts.provider, providerToolName,
+        toolUseId: opts.toolUseId, turnNumber: opts.turnNumber, status: 'error',
+        mcp: { serverName: opts.serverName, toolName: opts.toolName, invocation: 'not_invoked', outcome: opts.outcome },
+        arguments: { ...args, server: opts.serverName, tool: opts.toolName, status: opts.outcome },
+        summary: `MCP tool ${providerToolName} ${opts.outcome}`,
+        confidence: 'high',
+        rawSnippet: JSON.stringify({ status: opts.outcome }),
+    };
+}

package/dist/sdk/types.d.ts

@@ -5,6 +5,7 @@ import type { TrialResult } from '../types.js';
 import type { DiagnosticsReport } from './diagnostics.js';
 import type { LLMPort } from '../utils/llm-types.js';
 import type { McpSafetyOptions } from './mcp-safety.js';
+import type { McpMockApprovalRule } from './mcp-mock-approvals.js';
 export type AgentName = 'claude' | 'codex' | 'cursor' | 'opencode';
 export type AgentInteractionMode = 'prompt' | 'start_chat' | 'conversation';
 /** Runtime channel that actually executed the agent. */
@@ -28,6 +29,8 @@ export interface AgentOptions {
     copyFromHome?: string[];
     env?: Record<string, string>;
     mcpMock?: MockMcpServerDescriptor | MockMcpServerDescriptor[];
+    /** Ordered, persistent simulated-user decisions for approval-required generated MCP tools. */
+    mcpMockApprovalRules?: McpMockApprovalRule[];
     mcpConfigFile?: string;
     /** Configure the conversation window for transcript-based agents. Set false to disable. */
     conversationWindow?: ConversationWindowConfig | false;
@@ -352,7 +355,8 @@ export interface EvalResult {
     scorers: ScorerResultEntry[];
     tokenUsage?: TokenUsage;
 }
-export interface RecordedEvalResult extends EvalResult {
+export interface RecordedEvalResult extends Omit<EvalResult, 'score'> {
+    score?: number;
     trial?: TrialResult;
     resultKind?: EvaluationResultKind;
     scoringDurationMs?: number;
@@ -375,7 +379,7 @@ export interface ScorerResultEntry {
     errorCode?: string;
 }
 export interface PathgradeTestMeta {
-    score: number;
+    score?: number;
     scorers: ScorerResultEntry[];
     /** Stable identity for this evaluation definition across separate runs. */
     evaluationDefinitionKey?: string;

package/dist/tool-events.d.ts

@@ -1,4 +1,17 @@
-export type ToolAction = 'run_shell' | 'read_file' | 'write_file' | 'edit_file' | 'search_code' | 'list_files' | 'ask_user' | 'web_fetch' | 'use_skill' | 'update_todos' | 'mcp_tool_call' | 'unknown';
+export type ToolAction = 'run_shell' | 'read_file' | 'write_file' | 'edit_file' | 'search_code' | 'list_files' | 'ask_user' | 'web_fetch' | 'use_skill' | 'update_todos' | 'mcp_approval' | 'mcp_tool_call' | 'unknown';
+export type McpToolCallClassification = {
+    serverName: string;
+    toolName: string;
+} & ({
+    invocation: 'confirmed';
+    outcome: 'completed' | 'tool_error';
+} | {
+    invocation: 'not_invoked';
+    outcome: 'user_denied' | 'policy_denied' | 'protocol_error';
+} | {
+    invocation: 'unknown';
+    outcome: 'protocol_error' | 'unknown';
+});
 export interface ToolEvent {
     action: ToolAction;
     provider: 'claude' | 'codex' | 'cursor' | 'opencode';
@@ -9,6 +22,8 @@ export interface ToolEvent {
     arguments?: Record<string, unknown>;
     /** Lifecycle state observed by PathGrade. Absent on legacy provider events. */
     status?: 'completed' | 'error' | 'incomplete';
+    /** Receipt/enforcement-backed classification for canonical MCP calls. */
+    mcp?: McpToolCallClassification;
     /** Local wall-clock receive time for the invocation boundary. */
     startedAt?: string;
     /** Local wall-clock receive time for the matching result boundary. */

package/dist/types.d.ts

@@ -151,7 +151,7 @@ export interface LogEntry {
 export interface TrialResult {
     trial_id: number;
     name?: string;
-    reward: number;
+    reward?: number;
     scorer_results: ScorerResult[];
     duration_ms: number;
     n_commands: number;
@@ -385,6 +385,8 @@ export interface AgentSessionOptions {
     opencodeExecutable?: string;
     /** Exact generated MCP tool names accepted by the OpenCode event normalizer. */
     opencodeMcpToolNames?: string[];
+    /** Managed-session-owned runtime for scripted generated MCP. */
+    scriptedMcpHost?: import('./providers/scripted-mcp-mock-host.js').ScriptedMcpMockHost;
 }
 export declare abstract class BaseAgent {
     createSession(runtime: EnvironmentHandle, runCommand: AgentCommandRunner, options?: AgentSessionOptions): Promise<AgentSession>;

package/dist/viewer.html

@@ -997,9 +997,9 @@
     ${trials}
   `;
     }
-
     function renderTrial(t, idx) {
-      const pass = t.reward >= 0.5;
+      const evaluated = t.reward !== undefined;
+      const pass = evaluated && t.reward >= 0.5;
       const dur = ((t.duration_ms || 0) / 1000).toFixed(1);
       const tokens = (t.input_tokens || 0) + (t.output_tokens || 0);
       const convTokens = (t.conversation_input_tokens || 0) + (t.conversation_output_tokens || 0);
@@ -1026,8 +1026,8 @@
     <div class="trial-card" id="trial-${idx}">
       <div class="trial-header" onclick="toggleTrial(${idx})">
         <span class="trial-number">${t.name ? esc(t.name) : 'Trial ' + t.trial_id}</span>
-        <span class="trial-reward ${pass ? 'pass' : 'fail'}">${t.reward.toFixed(2)}</span>
-        <span class="badge ${pass ? 'badge-pass' : 'badge-fail'}">${pass ? 'PASS' : 'FAIL'}</span>
+        <span class="trial-reward ${pass ? 'pass' : 'fail'}">${evaluated ? t.reward.toFixed(2) : 'n/a'}</span>
+        <span class="badge ${pass ? 'badge-pass' : 'badge-fail'}">${evaluated ? (pass ? 'PASS' : 'FAIL') : 'N/A'}</span>
         <span class="trial-meta">
           <span>${dur}s</span>
           <span>${t.n_commands || 0} cmds</span>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wix/pathgrade",
-  "version": "1.0.13",
+  "version": "1.0.15",
   "packageManager": "yarn@4.12.0",
   "description": "Evaluate whether AI agents discover and use your skills correctly",
   "exports": {
@@ -125,6 +125,7 @@
     "@anthropic-ai/claude-agent-sdk": "0.2.141",
     "@modelcontextprotocol/sdk": "1.30.0",
     "@types/node": "25.6.0",
+    "ajv": "8.20.0",
     "fs-extra": "11.3.3",
     "jiti": "2.6.1",
     "picomatch": "^4.0.4",
@@ -132,5 +133,5 @@
     "typescript": "^5.9.3",
     "zod": "4.3.6"
   },
-  "falconPackageHash": "b327de99650979dcf1db770211744d11c34f2555b5f0b82869b80d30"
+  "falconPackageHash": "7c3967f299416b4436afa2b39609adc26ca8dbd4079ffeb875453235"
 }