@wix/pathgrade 1.0.37 → 1.0.39

package/dist/sdk/lifecycle.d.ts

@@ -1,4 +1,4 @@
-import type { Agent, PathgradeTestMeta, RecordedEvalResult } from './types.js';
+import type { Agent, AgentFlowTrace, PathgradeTestMeta, RecordedEvalResult } from './types.js';
 export type LifecycleAgentOwner = {
     type: 'runner-case';
     caseId: string;
@@ -24,6 +24,7 @@ declare function registerAgent(agent: Agent, owner?: LifecycleAgentOwner | null)
 declare function untrackAgent(agent: Agent): void;
 declare function releaseAgent(agent: Agent): void;
 declare function recordResult(result: RecordedEvalResult, agent: Agent, attribution?: ResultAttribution): void;
+declare function recordFlowResult(result: RecordedEvalResult, flow: AgentFlowTrace, caseId?: string): void;
 declare function flushCase(input: FlushCaseInput): Promise<PathgradeTestMeta[]>;
 declare function cleanupAll(): Promise<void>;
 declare function reset(): void;
@@ -33,6 +34,7 @@ export declare const lifecycleCore: {
     releaseAgent: typeof releaseAgent;
     getAgentOwner: typeof getAgentOwner;
     recordResult: typeof recordResult;
+    recordFlowResult: typeof recordFlowResult;
     flushCase: typeof flushCase;
     cleanupAll: typeof cleanupAll;
     reset: typeof reset;

package/dist/sdk/lifecycle.js

@@ -1,14 +1,19 @@
 import { getCurrentCaseContext } from './case-context.js';
 import { buildDiagnosticsReport } from './diagnostics.js';
 import { countShellCommandsFromLog } from '../tool-events.js';
+import { summarizeFlow } from './agent-flow.js';
 const lifecycleStateKey = Symbol.for('@wix/pathgrade/lifecycle-state');
 const sharedLifecycle = globalThis;
 const lifecycleState = sharedLifecycle[lifecycleStateKey] ?? (sharedLifecycle[lifecycleStateKey] = {
     pendingAgents: new Set(),
     agentOwners: new WeakMap(),
     agentResults: new WeakMap(),
+    flowResults: new Map(),
+    nextResultSequence: 0,
 });
-const { pendingAgents, agentOwners, agentResults } = lifecycleState;
+lifecycleState.nextResultSequence ??= 0;
+lifecycleState.flowResults ??= new Map();
+const { pendingAgents, agentOwners, agentResults, flowResults } = lifecycleState;
 function currentAgentOwner() {
     const current = getCurrentCaseContext();
     if (current.status !== 'active')
@@ -74,6 +79,7 @@ function recordResult(result, agent, attribution) {
     const conversationEnd = [...agent.log].reverse().find((entry) => entry.type === 'conversation_end');
     const completionReason = conversationEnd?.completion_reason ?? (agent.log.some((entry) => entry.type === 'agent_result') ? 'completed' : undefined);
     agentResults.get(agent).push({
+        sequence: lifecycleState.nextResultSequence++,
         attribution: attribution ?? currentResultAttribution(owner),
         meta: {
             score: result.score,
@@ -101,8 +107,60 @@ function recordResult(result, agent, attribution) {
         },
     });
 }
+function recordFlowResult(result, flow, caseId) {
+    const current = getCurrentCaseContext();
+    const resolvedCaseId = caseId ?? (current.status === 'active' && current.context.scope === 'runner-case'
+        ? current.context.caseId
+        : undefined);
+    if (!resolvedCaseId)
+        return;
+    const entries = flowResults.get(resolvedCaseId) ?? [];
+    entries.push({
+        sequence: lifecycleState.nextResultSequence++,
+        meta: {
+            score: result.score,
+            scorers: result.scorers,
+            ...(result.evaluationDefinitionKey ? { evaluationDefinitionKey: result.evaluationDefinitionKey } : {}),
+            ...(result.scorerRevision ? { scorerRevision: result.scorerRevision } : {}),
+            trial: {
+                ...(result.trial ?? {
+                    trial_id: 0,
+                    ...(result.score !== undefined ? { reward: result.score } : {}),
+                    scorer_results: result.scorers.map(scorer => ({
+                        scorer_type: scorer.type === 'check' || scorer.type === 'score' ? 'deterministic' : scorer.type === 'judge' ? 'llm_rubric' : 'tool_usage',
+                        score: scorer.score,
+                        weight: scorer.weight,
+                        details: scorer.details ?? '',
+                        status: scorer.status,
+                    })),
+                    duration_ms: 0,
+                    n_commands: 0,
+                    input_tokens: result.tokenUsage?.inputTokens ?? 0,
+                    output_tokens: result.tokenUsage?.outputTokens ?? 0,
+                    session_log: [],
+                }),
+                flow_summary: summarizeFlow(flow),
+                flow_trace: structuredClone(flow),
+            },
+            resultKind: result.resultKind ?? 'evaluated',
+            ...(result.scoringDurationMs !== undefined ? { scoringDurationMs: result.scoringDurationMs } : {}),
+            ...(result.recordedAt ? { recordedAt: result.recordedAt } : {}),
+            diagnostics: buildDiagnosticsReport({
+                completionReason: ['completed', 'failed', 'rejected', 'canceled', 'auth-required'].includes(flow.outcome.state)
+                    ? flow.outcome.state
+                    : undefined,
+                completionDetail: flow.outcome.error,
+                score: result.score,
+                scorers: result.scorers,
+                log: [],
+            }),
+        },
+    });
+    flowResults.set(resolvedCaseId, entries);
+}
 async function flushCase(input) {
-    const results = [];
+    const results = flowResults.get(input.caseId) ?? [];
+    flowResults.delete(input.caseId);
     const toDispose = [];
     for (const agent of pendingAgents) {
         const owner = agentOwners.get(agent);
@@ -110,7 +168,7 @@ async function flushCase(input) {
         const meta = agentResults.get(agent);
         const matchingMeta = meta?.filter((entry) => resultMatchesCase(entry, input.caseId));
         if (matchingMeta && matchingMeta.length > 0 && canFlushResultsToCase(owner, input.caseId)) {
-            results.push(...matchingMeta.map((entry) => entry.meta));
+            results.push(...matchingMeta);
             const remainingMeta = meta?.filter((entry) => !resultMatchesCase(entry, input.caseId)) ?? [];
             if (remainingMeta.length > 0) {
                 agentResults.set(agent, remainingMeta);
@@ -124,7 +182,7 @@ async function flushCase(input) {
             if (!hasResultsForCase) {
                 const synthTrial = synthesizeTrialFromAgent(agent);
                 if (synthTrial) {
-                    results.push(synthTrial);
+                    results.push({ sequence: lifecycleState.nextResultSequence++, meta: synthTrial });
                 }
             }
             pendingAgents.delete(agent);
@@ -133,7 +191,7 @@ async function flushCase(input) {
         }
     }
     await Promise.all(toDispose.map((agent) => agent.dispose().catch(() => { })));
-    return results;
+    return results.toSorted((left, right) => left.sequence - right.sequence).map(entry => entry.meta);
 }
 function synthesizeTrialFromAgent(agent) {
     if (agent.log.length === 0)
@@ -181,10 +239,14 @@ function synthesizeTrialFromAgent(agent) {
 async function cleanupAll() {
     const toDispose = [...pendingAgents];
     pendingAgents.clear();
+    flowResults.clear();
+    lifecycleState.nextResultSequence = 0;
     await Promise.all(toDispose.map((agent) => agent.dispose().catch(() => { })));
 }
 function reset() {
     pendingAgents.clear();
+    flowResults.clear();
+    lifecycleState.nextResultSequence = 0;
 }
 export const lifecycleCore = {
     registerAgent,
@@ -192,6 +254,7 @@ export const lifecycleCore = {
     releaseAgent,
     getAgentOwner,
     recordResult,
+    recordFlowResult,
     flushCase,
     cleanupAll,
     reset,

package/dist/sdk/mcp-event-input.d.ts

@@ -1,3 +1,4 @@
 import type { ToolEvent } from '../tool-events.js';
 export declare function attachOriginalMcpInput(event: ToolEvent, input: Record<string, unknown>): ToolEvent;
 export declare function getOriginalMcpInput(event: ToolEvent): Record<string, unknown> | undefined;
+export declare function clearOriginalMcpInput(event: ToolEvent): void;

package/dist/sdk/mcp-event-input.js

@@ -6,3 +6,6 @@ export function attachOriginalMcpInput(event, input) {
 export function getOriginalMcpInput(event) {
     return originalInputs.get(event);
 }
+export function clearOriginalMcpInput(event) {
+    originalInputs.delete(event);
+}

package/dist/sdk/mcp-evidence.js

@@ -2,9 +2,12 @@ export function getMcpToolCall(event) {
     if (event.action !== 'mcp_tool_call')
         return undefined;
     const args = event.arguments ?? {};
-    const serverName = typeof args.server === 'string' ? args.server : undefined;
-    const toolName = typeof args.tool === 'string' ? args.tool : undefined;
-    const status = typeof args.status === 'string' ? args.status : undefined;
+    const serverName = event.mcp?.serverName
+        ?? (typeof args.server === 'string' ? args.server : undefined);
+    const toolName = event.mcp?.toolName
+        ?? (typeof args.tool === 'string' ? args.tool : undefined);
+    const argumentStatus = typeof args.status === 'string' ? args.status : undefined;
+    const status = mcpClassificationStatus(event.mcp, argumentStatus) ?? argumentStatus;
     if (!serverName || !toolName || !status)
         return undefined;
     return {
@@ -15,6 +18,16 @@ export function getMcpToolCall(event) {
         event,
     };
 }
+function mcpClassificationStatus(mcp, argumentStatus) {
+    if (!mcp)
+        return undefined;
+    if (mcp.invocation === 'confirmed') {
+        if (mcp.outcome === 'completed')
+            return 'completed';
+        return argumentStatus === 'failed' || argumentStatus === 'error' ? argumentStatus : 'error';
+    }
+    return mcp.outcome;
+}
 export function isMcpToolCall(event, expected = {}) {
     const call = getMcpToolCall(event);
     return call !== undefined && matchesMcpToolCall(call, expected);

package/dist/sdk/mcp-safety.js

@@ -1,4 +1,4 @@
-import { sanitizePersistenceValue } from '../tool-event-results.js';
+import { sanitizeUntrustedPersistenceValue } from '../tool-event-results.js';
 export function decideMcpToolCall(options, request) {
     const runMode = options?.runMode ?? 'mock';
     if (runMode === 'mock')
@@ -31,7 +31,7 @@ export function decideMcpToolCall(options, request) {
     return { action: 'allow' };
 }
 export function redactMcpSecrets(value) {
-    return sanitizePersistenceValue(value);
+    return sanitizeUntrustedPersistenceValue(value);
 }
 function ruleMatches(rule, request) {
     if (rule.serverName !== undefined && rule.serverName !== request.serverName)

package/dist/sdk/result-capture.d.ts

@@ -1,4 +1,4 @@
-import type { Agent, RecordedEvalResult } from './types.js';
+import type { Agent, AgentFlowTrace, RecordedEvalResult } from './types.js';
 import { type CaseContext } from './case-context.js';
 export interface EvalResultEvent {
     readonly result: RecordedEvalResult;
@@ -6,6 +6,15 @@ export interface EvalResultEvent {
     readonly case?: CaseContext;
 }
 export type EvalResultObserver = (event: EvalResultEvent) => void;
+export type EvaluationResultEvent = (EvalResultEvent & {
+    readonly sourceKind: 'agent';
+}) | {
+    readonly sourceKind: 'flow';
+    readonly result: RecordedEvalResult;
+    readonly flow: AgentFlowTrace;
+    readonly case?: CaseContext;
+};
+export type EvaluationResultObserver = (event: EvaluationResultEvent) => void;
 export type ResultObserverOwner = 'user' | 'adapter' | 'test';
 export interface ResultObserverOptions {
     readonly owner?: ResultObserverOwner;
@@ -15,6 +24,10 @@ export interface ResultObserverHandle {
     unsubscribe(): void;
 }
 export declare function subscribeToEvalResults(observer: EvalResultObserver, options?: ResultObserverOptions): ResultObserverHandle;
+export declare function subscribeToEvaluationResults(observer: EvaluationResultObserver, options?: ResultObserverOptions): ResultObserverHandle;
 export declare function emitEvalResult(event: EvalResultEvent): void;
+export declare function emitFlowEvalResult(event: Omit<Extract<EvaluationResultEvent, {
+    sourceKind: 'flow';
+}>, 'sourceKind'>): void;
 export declare function resetUserResultObservers(): void;
 export declare function resetAllResultObserversForTests(): void;

package/dist/sdk/result-capture.js

@@ -3,6 +3,10 @@ const observerRegistryKey = Symbol.for('@wix/pathgrade/eval-result-observers');
 const globalRegistry = globalThis;
 const observers = globalRegistry[observerRegistryKey]
     ?? (globalRegistry[observerRegistryKey] = new Set());
+const evaluationObserverRegistryKey = Symbol.for('@wix/pathgrade/evaluation-result-observers');
+const evaluationGlobalRegistry = globalThis;
+const evaluationObservers = evaluationGlobalRegistry[evaluationObserverRegistryKey]
+    ?? (evaluationGlobalRegistry[evaluationObserverRegistryKey] = new Set());
 export function subscribeToEvalResults(observer, options = {}) {
     if (options.owner === 'adapter' && options.key) {
         removeObserverByOwnerAndKey(options.owner, options.key);
@@ -23,6 +27,22 @@ export function subscribeToEvalResults(observer, options = {}) {
         },
     };
 }
+export function subscribeToEvaluationResults(observer, options = {}) {
+    if (options.owner === 'adapter' && options.key) {
+        removeEvaluationObserverByOwnerAndKey(options.owner, options.key);
+    }
+    const subscription = { observer, owner: options.owner ?? 'user', key: options.key };
+    evaluationObservers.add(subscription);
+    let active = true;
+    return {
+        unsubscribe() {
+            if (!active)
+                return;
+            active = false;
+            evaluationObservers.delete(subscription);
+        },
+    };
+}
 export function emitEvalResult(event) {
     const currentCase = getCurrentCaseContext();
     const deliveredEvent = event.case || currentCase.status !== 'active'
@@ -40,6 +60,14 @@ export function emitEvalResult(event) {
             // broken hook must not block adapter-owned reporting.
         }
     }
+    deliverEvaluationEvent({ ...deliveredEvent, sourceKind: 'agent' });
+}
+export function emitFlowEvalResult(event) {
+    const currentCase = getCurrentCaseContext();
+    const deliveredEvent = event.case || currentCase.status !== 'active'
+        ? event
+        : { ...event, case: currentCase.context };
+    deliverEvaluationEvent({ ...deliveredEvent, sourceKind: 'flow' });
 }
 export function resetUserResultObservers() {
     for (const subscription of observers) {
@@ -47,9 +75,14 @@ export function resetUserResultObservers() {
             observers.delete(subscription);
         }
     }
+    for (const subscription of evaluationObservers) {
+        if (subscription.owner === 'user')
+            evaluationObservers.delete(subscription);
+    }
 }
 export function resetAllResultObserversForTests() {
     observers.clear();
+    evaluationObservers.clear();
 }
 function removeObserverByOwnerAndKey(owner, key) {
     for (const subscription of observers) {
@@ -58,3 +91,19 @@ function removeObserverByOwnerAndKey(owner, key) {
         }
     }
 }
+function removeEvaluationObserverByOwnerAndKey(owner, key) {
+    for (const subscription of evaluationObservers) {
+        if (subscription.owner === owner && subscription.key === key)
+            evaluationObservers.delete(subscription);
+    }
+}
+function deliverEvaluationEvent(event) {
+    for (const subscription of [...evaluationObservers]) {
+        try {
+            subscription.observer(event);
+        }
+        catch {
+            // Generic result capture has the same best-effort isolation as the legacy stream.
+        }
+    }
+}

package/dist/sdk/scripted-mcp-events.js

@@ -1,4 +1,5 @@
 import { redactMcpSecrets } from './mcp-safety.js';
+import { attachLiveMcpInput } from './tool-event-secrets.js';
 export function buildScriptedMcpApprovalEvent(opts) {
     const providerToolName = `${opts.serverName}.${opts.toolName}`;
     return {
@@ -18,7 +19,7 @@ export function buildScriptedMcpApprovalEvent(opts) {
 export function buildScriptedMcpDeniedCallEvent(opts) {
     const providerToolName = `${opts.serverName}.${opts.toolName}`;
     const args = redactMcpSecrets(opts.args);
-    return {
+    return attachLiveMcpInput({
         action: 'mcp_tool_call', provider: opts.provider, providerToolName,
         toolUseId: opts.toolUseId, turnNumber: opts.turnNumber, status: 'error',
         mcp: { serverName: opts.serverName, toolName: opts.toolName, invocation: 'not_invoked', outcome: opts.outcome },
@@ -26,5 +27,5 @@ export function buildScriptedMcpDeniedCallEvent(opts) {
         summary: `MCP tool ${providerToolName} ${opts.outcome}`,
         confidence: 'high',
         rawSnippet: JSON.stringify({ status: opts.outcome }),
-    };
+    }, opts.args);
 }

package/dist/sdk/tool-event-log.js

@@ -1,7 +1,22 @@
-import { sanitizePersistenceValue } from '../tool-event-results.js';
-import { getToolEventSensitiveValues } from './tool-event-secrets.js';
+import { collectStructuredSensitiveValues, isSensitiveValueScanLimitError, sanitizePersistenceValue, } from '../tool-event-results.js';
+import { attachToolEventSensitiveValues, cloneToolEventWithRuntimeMetadata, getToolEventSensitiveValues, redactToolEventPayload, } from './tool-event-secrets.js';
 export function buildToolEventLogEntry(toolEvent, fallbackTimestamp) {
-    const persistedEvent = sanitizePersistenceValue(toolEvent, getToolEventSensitiveValues(toolEvent));
+    let event = toolEvent;
+    let discoveredValues;
+    try {
+        discoveredValues = collectStructuredSensitiveValues(event);
+    }
+    catch (error) {
+        if (!isSensitiveValueScanLimitError(error))
+            throw error;
+        event = redactToolEventPayload(event);
+        discoveredValues = [];
+    }
+    const sensitiveValues = [...new Set([
+            ...getToolEventSensitiveValues(toolEvent),
+            ...discoveredValues,
+        ])];
+    const persistedEvent = attachToolEventSensitiveValues(cloneToolEventWithRuntimeMetadata(event, sanitizePersistenceValue(event, sensitiveValues)), sensitiveValues);
     return {
         type: 'tool_event',
         timestamp: persistedEvent.startedAt ?? fallbackTimestamp,

package/dist/sdk/tool-event-secrets.d.ts

@@ -1,4 +1,8 @@
 import type { ToolEvent } from '../tool-events.js';
 export declare function attachToolEventSensitiveValues(event: ToolEvent, sensitiveValues: readonly string[]): ToolEvent;
 export declare function getToolEventSensitiveValues(event: ToolEvent): readonly string[];
+export declare function attachLiveMcpInput(event: ToolEvent, input: Record<string, unknown>): ToolEvent;
+export declare function redactToolEventPayload(event: ToolEvent): ToolEvent;
+export declare function collectToolEventSensitiveValues(events: readonly ToolEvent[]): string[];
+export declare function clearToolEventRuntimeMetadata(event: ToolEvent): void;
 export declare function cloneToolEventWithRuntimeMetadata(source: ToolEvent, overrides: Partial<ToolEvent>): ToolEvent;

package/dist/sdk/tool-event-secrets.js

@@ -1,12 +1,43 @@
-import { attachOriginalMcpInput, getOriginalMcpInput } from './mcp-event-input.js';
+import { collectStructuredSensitiveValues, isSensitiveValueScanLimitError } from '../tool-event-results.js';
+import { attachOriginalMcpInput, clearOriginalMcpInput, getOriginalMcpInput, } from './mcp-event-input.js';
 const sensitiveValuesByEvent = new WeakMap();
 export function attachToolEventSensitiveValues(event, sensitiveValues) {
-    sensitiveValuesByEvent.set(event, [...sensitiveValues]);
+    sensitiveValuesByEvent.set(event, [...new Set([
+            ...getToolEventSensitiveValues(event),
+            ...sensitiveValues,
+        ])]);
     return event;
 }
 export function getToolEventSensitiveValues(event) {
     return sensitiveValuesByEvent.get(event) ?? [];
 }
+export function attachLiveMcpInput(event, input) {
+    let sensitiveValues;
+    try {
+        sensitiveValues = collectStructuredSensitiveValues(input);
+    }
+    catch (error) {
+        if (!isSensitiveValueScanLimitError(error))
+            throw error;
+        return redactToolEventPayload(event);
+    }
+    return attachToolEventSensitiveValues(attachOriginalMcpInput(event, input), sensitiveValues);
+}
+export function redactToolEventPayload(event) {
+    return {
+        ...event,
+        arguments: { redacted: true },
+        summary: `${event.action} via ${event.providerToolName}`,
+        rawSnippet: '<redacted>',
+    };
+}
+export function collectToolEventSensitiveValues(events) {
+    return [...new Set(events.flatMap((event) => getToolEventSensitiveValues(event)))];
+}
+export function clearToolEventRuntimeMetadata(event) {
+    sensitiveValuesByEvent.delete(event);
+    clearOriginalMcpInput(event);
+}
 export function cloneToolEventWithRuntimeMetadata(source, overrides) {
     const clone = attachToolEventSensitiveValues({ ...source, ...overrides }, getToolEventSensitiveValues(source));
     const originalInput = getOriginalMcpInput(source);

package/dist/sdk/types.d.ts

@@ -8,6 +8,8 @@ import type { McpSafetyOptions } from './mcp-safety.js';
 import type { McpMockApprovalRule } from './mcp-mock-approvals.js';
 import type { EvidenceEnvelope, JsonValue } from '../internal/direct-mcp-v2/types.js';
 import type { ScenarioMachineV2 } from './scenario-machine-v2.js';
+import type { AgentFlowTrace } from './agent-flow.js';
+export type { AgentFlowTrace, AgentFlowParticipant, AgentFlowInteraction, AgentFlowEvidenceAvailability, AgentFlowEvidenceCompleteness, AgentFlowSummary } from './agent-flow.js';
 export type AgentName = 'claude' | 'codex' | 'cursor' | 'opencode';
 export type AgentInteractionMode = 'prompt' | 'start_chat' | 'conversation';
 /** Runtime channel that actually executed the agent. */
@@ -351,11 +353,15 @@ export interface ScorerContext {
     artifacts: SessionArtifacts;
     /** Canonical host evidence for ScenarioMachineV2 scorers. */
     scenarioEvidence?: readonly EvidenceEnvelope[];
+    /** Protocol-neutral multi-agent execution evidence, when evaluating a flow. */
+    flow?: AgentFlowTrace;
 }
 export interface EvaluateOptions {
     failFast?: boolean;
     llm?: LLMPort;
     onScorerError?: 'skip' | 'zero' | 'fail';
+    /** Evidence visible to deterministic scorers. Live mode is unavailable for snapshot replay. */
+    deterministicToolEvidence?: 'persisted' | 'live';
     /** Stable identity for this evaluation definition across separate runs. */
     evaluationDefinitionKey?: string;
 }

package/dist/tool-event-results.d.ts

@@ -7,4 +7,7 @@ export declare function collectSensitiveEnvValues(env?: Readonly<Record<string,
  * is important for summaries, snippets, traces, and provider error text.
  */
 export declare function sanitizePersistenceValue<T>(source: T, explicitSensitiveValues?: readonly string[]): T;
+export declare function sanitizeUntrustedPersistenceValue<T>(source: T, explicitSensitiveValues?: readonly string[]): T;
+export declare function isSensitiveValueScanLimitError(error: unknown): boolean;
 export declare function sanitizeToolEventResult(source: Readonly<ToolEventResult>, sensitiveValues: readonly string[]): ToolEventResult;
+export declare function collectStructuredSensitiveValues(source: unknown): string[];