@wix/pathgrade 1.0.37 → 1.0.39

package/dist/reporting/report-parser.js

@@ -6,18 +6,44 @@ export function parsePathgradeReport(value) {
         || (value.version === 2 && (typeof value.overall_mean_reward !== 'number' || !validAttemptCounts(value)))
         || !optionalNumber(value.threshold)
         || (value.status !== 'pass' && value.status !== 'fail')
+        || !optionalGateStatus(value.runner_status)
+        || !optionalThresholdStatus(value.threshold_status)
         || (value.run_kind !== undefined && value.run_kind !== 'evaluation' && value.run_kind !== 'no-affected')
         || !Array.isArray(value.groups)
         || !value.groups.every(group => isReportGroup(group, value.version))
         || !isSelection(value.selection)) {
         throw new Error('PathGrade results.json is missing or has an unsupported schema');
     }
-    return value;
+    const threshold = value.threshold;
+    const groups = value.groups.map(group => {
+        const runnerStatus = deriveRunnerStatus(group);
+        const thresholdStatus = deriveThresholdStatus(group.mean_reward ?? group.pass_rate, threshold);
+        return {
+            ...group,
+            runner_status: runnerStatus,
+            threshold_status: thresholdStatus,
+            status: composeStatus(runnerStatus, thresholdStatus),
+        };
+    });
+    const legacyFailure = value.runner_status === undefined && value.status === 'fail' && groups.length === 0;
+    const runnerStatus = value.runner_status !== 'fail' && !legacyFailure
+        && (groups.length > 0 || value.runner_status === 'pass' || value.status === 'pass')
+        && groups.every(group => group.runner_status === 'pass') ? 'pass' : 'fail';
+    const thresholdStatus = deriveThresholdStatus(value.overall_mean_reward ?? value.overall_pass_rate, threshold);
+    return {
+        ...value,
+        groups,
+        runner_status: runnerStatus,
+        threshold_status: thresholdStatus,
+        status: composeStatus(runnerStatus, thresholdStatus),
+    };
 }
 function isReportGroup(value, version) {
     return isRecord(value)
         && typeof value.task === 'string'
         && (version === 1 ? legacyMetrics(value) : schemaV2Metrics(value))
+        && optionalGateStatus(value.runner_status)
+        && optionalThresholdStatus(value.threshold_status)
         && optionalString(value.source_file)
         && typeof value.trace_file === 'string'
         && Array.isArray(value.skills_used)
@@ -26,6 +52,34 @@ function isReportGroup(value, version) {
         && value.trials.every(isTrial)
         && (value.comparison_contract === undefined || isComparisonContract(value.comparison_contract));
 }
+function deriveRunnerStatus(value) {
+    if (value.runner_status === 'fail')
+        return 'fail';
+    const outcomes = value.trials
+        .map(trial => trial.runner_outcome)
+        .filter(outcome => outcome !== undefined);
+    if (outcomes.length > 0)
+        return outcomes.every(outcome => outcome === 'passed') ? 'pass' : 'fail';
+    if (value.runner_status === 'pass')
+        return value.runner_status;
+    if (value.status === 'pass' || value.status === 'fail')
+        return value.status;
+    return value.pass_rate === 1 ? 'pass' : 'fail';
+}
+function deriveThresholdStatus(reward, threshold) {
+    if (threshold === undefined)
+        return 'not_configured';
+    return typeof reward === 'number' && reward >= threshold ? 'pass' : 'fail';
+}
+function composeStatus(runnerStatus, thresholdStatus) {
+    return runnerStatus === 'fail' || thresholdStatus === 'fail' ? 'fail' : 'pass';
+}
+function optionalGateStatus(value) {
+    return value === undefined || value === 'pass' || value === 'fail';
+}
+function optionalThresholdStatus(value) {
+    return optionalGateStatus(value) || value === 'not_configured';
+}
 function isTrial(value) {
     return isRecord(value)
         && typeof value.trial_id === 'number'
@@ -40,7 +94,8 @@ function isTrial(value) {
         && value.scorer_results.every(scorer => isRecord(scorer)
             && typeof scorer.scorer_type === 'string'
             && typeof scorer.score === 'number'
-            && typeof scorer.weight === 'number');
+            && typeof scorer.weight === 'number'
+            && (scorer.status === undefined || scorer.status === 'ok' || scorer.status === 'error' || scorer.status === 'skipped'));
 }
 function legacyMetrics(value) {
     return typeof value.pass_rate === 'number'

package/dist/reporting/types.d.ts

@@ -1,9 +1,10 @@
 import type { DiagnosticsReport } from '../sdk/diagnostics.js';
 import type { AgentExecutionMetadata, EvaluationResultKind } from '../sdk/types.js';
-import type { AttemptOutcome } from '../runners/model.js';
+import type { AttemptOutcome, RunStatus } from '../runners/model.js';
 import type { PathgradeReport, PathgradeSelectionReport, TrialResult } from '../types.js';
 export type ReportCaseState = 'passed' | 'failed' | 'skipped' | 'pending';
 export interface ReportRunInput {
+    runStatus?: RunStatus;
     threshold?: number;
     selection?: PathgradeSelectionReport;
     attemptsRequested?: number;

package/dist/runners/adapter.d.ts

@@ -1,5 +1,5 @@
 import type { PathgradeSelectionReport } from '../types.js';
-import type { Agent, PathgradeTestMeta, RecordedEvalResult } from '../sdk/types.js';
+import type { Agent, AgentFlowTrace, PathgradeTestMeta, RecordedEvalResult } from '../sdk/types.js';
 import type { CaseContextScope } from '../sdk/case-context.js';
 import type { NormalizedRunSnapshot } from './model.js';
 export declare const runnerAdapterContractVersion = 1;
@@ -61,5 +61,7 @@ export interface AdapterCaseContext {
 }
 export interface EvalResultEvent {
     result: RecordedEvalResult;
-    agent: Agent;
+    agent?: Agent;
+    flow?: AgentFlowTrace;
+    case?: import('../sdk/case-context.js').CaseContext;
 }

package/dist/runners/lifecycle-hooks.js

@@ -2,7 +2,12 @@ import { lifecycleCore } from '../sdk/lifecycle.js';
 import { runWithCaseContext } from '../sdk/case-context.js';
 export function createRunnerLifecycleHooks() {
     return {
-        onResult: event => lifecycleCore.recordResult(event.result, event.agent),
+        onResult: event => {
+            if (event.flow)
+                lifecycleCore.recordFlowResult(event.result, event.flow, event.case?.caseId);
+            else if (event.agent)
+                lifecycleCore.recordResult(event.result, event.agent);
+        },
         withCaseContext: (context, run) => runWithCaseContext(toCaseContext(context), run),
         flushCase: caseId => lifecycleCore.flushCase({ caseId }),
         cleanupRun: () => lifecycleCore.cleanupAll(),

package/dist/runners/orchestrator.js

@@ -43,8 +43,8 @@ export async function runWithAdapter(input) {
         if (loadedSelection) {
             built = buildPathgradeReport({
                 threshold: options.threshold,
+                ...reportInput,
                 selection: loadedSelection,
-                groups: reportInput.groups,
             });
         }
         const mode = options.reporterMode ?? 'cli';
@@ -56,14 +56,13 @@ export async function runWithAdapter(input) {
         if (mode === 'browser') {
             await options.openBrowser?.();
         }
-        if (options.threshold != null && built.report.status === 'fail') {
+        if (options.threshold != null && built.report.threshold_status === 'fail') {
             options.onThresholdFailure?.({
                 overallPassRate: built.report.overall_pass_rate,
                 threshold: options.threshold,
             });
-            return runExitCode === 0 ? 1 : runExitCode;
         }
-        return runExitCode;
+        return built.report.status === 'fail' && runExitCode === 0 ? 1 : runExitCode;
     }
     finally {
         await lifecycle.cleanupRun();

package/dist/runners/repeated-invocation.js

@@ -80,13 +80,10 @@ export function withRepeatedAttempts(input) {
                     if (input.config.reporter === 'browser') {
                         await (input.openBrowser ?? openBrowserViewer)();
                     }
-                    if (input.config.ci.threshold != null && built.report.status === 'fail') {
+                    if (input.config.ci.threshold != null && built.report.threshold_status === 'fail') {
                         process.stdout.write(`\n  CI THRESHOLD FAILED  avg score ${built.report.overall_pass_rate.toFixed(3)} < threshold ${input.config.ci.threshold}\n\n`);
-                        resultCode = firstNonzeroExit || 1;
-                    }
-                    else {
-                        resultCode = firstNonzeroExit;
                     }
+                    resultCode = firstNonzeroExit || (built.report.status === 'fail' ? 1 : 0);
                 }
             }
             catch (error) {

package/dist/runners/report-projection.js

@@ -41,6 +41,7 @@ export function projectNormalizedRunSnapshotToReportInput(snapshot, options = {}
     }
     return {
         ...(options.selection ? { selection: options.selection } : {}),
+        runStatus: snapshot.model.run.status,
         attemptsRequested: snapshot.model.run.attemptsRequested ?? 1,
         attemptsCompleted: snapshot.model.run.attemptsCompleted ?? 1,
         groups: [...groups.values()],

package/dist/runners/vitest-lifecycle.d.ts

@@ -6,6 +6,7 @@ export type VitestAfterEachFn = (fn: (ctx: {
     task: {
         id: string;
         meta: Record<string, unknown>;
+        suite?: unknown;
     };
 }) => Promise<void>) => void;
 export type VitestAfterAllFn = (fn: () => Promise<void>) => void;

package/dist/runners/vitest-lifecycle.js

@@ -1,10 +1,15 @@
 import path from 'node:path';
-import { subscribeToEvalResults } from '../sdk/result-capture.js';
+import { subscribeToEvaluationResults } from '../sdk/result-capture.js';
 import { getCurrentCaseContext, installCaseContextProvider, runWithCaseContext, } from '../sdk/case-context.js';
 import { lifecycleCore } from '../sdk/lifecycle.js';
 export function createVitestLifecycleHooks() {
     return {
-        onResult: event => recordVitestResult(event.result, event.agent),
+        onResult: event => {
+            if (event.flow)
+                recordVitestFlowResult(event.result, event.flow, event.case);
+            else if (event.agent)
+                recordVitestResult(event.result, event.agent);
+        },
         withCaseContext: (context, run) => runWithCaseContext(toCaseContext(context), run),
         flushCase: caseId => lifecycleCore.flushCase({ caseId }),
         cleanupRun: () => lifecycleCore.cleanupAll(),
@@ -18,7 +23,11 @@ export function installVitestLifecycle(input) {
     let resultsUnsubscribed = false;
     input.aroundEach?.(async (runTest, { task }) => lifecycle.withCaseContext(caseContextForTask(task), runTest));
     input.afterEach(async ({ task }) => {
-        const results = await lifecycle.flushCase(task.id);
+        const filePath = filePathForTask(task);
+        const results = [
+            ...(filePath ? await lifecycle.flushCase(`file:${filePath}`) : []),
+            ...await lifecycle.flushCase(task.id),
+        ];
         if (results.length > 0) {
             task.meta.pathgrade = results;
         }
@@ -69,7 +78,9 @@ export function resetVitestLifecycle(handle) {
     handle?.restore();
 }
 function defaultSubscribeToResults(callback) {
-    return subscribeToEvalResults(({ result, agent }) => callback({ result, agent }), { owner: 'adapter', key: 'vitest-lifecycle' });
+    return subscribeToEvaluationResults(event => callback(event.sourceKind === 'flow'
+        ? { result: event.result, flow: event.flow, case: event.case }
+        : { result: event.result, agent: event.agent, case: event.case }), { owner: 'adapter', key: 'vitest-lifecycle' });
 }
 function defaultInstallFileContextProvider() {
     return installCaseContextProvider(currentFileContext);
@@ -148,3 +159,9 @@ function recordVitestResult(result, agent) {
     }
     lifecycleCore.recordResult(result, agent);
 }
+function recordVitestFlowResult(result, flow, capturedCase) {
+    const caseId = capturedCase?.scope === 'runner-case'
+        ? capturedCase.caseId
+        : currentTaskId() || capturedCase?.caseId;
+    lifecycleCore.recordFlowResult(result, flow, caseId || undefined);
+}

package/dist/sdk/agent-flow.d.ts

@@ -0,0 +1,51 @@
+export type AgentFlowEvidenceAvailability = 'complete' | 'partial' | 'unavailable';
+export interface AgentFlowEvidenceCompleteness {
+    topology: AgentFlowEvidenceAvailability;
+    outcomes: AgentFlowEvidenceAvailability;
+    timing: AgentFlowEvidenceAvailability;
+    usage: AgentFlowEvidenceAvailability;
+    runtimeIdentity: AgentFlowEvidenceAvailability;
+}
+export interface AgentFlowParticipant {
+    id: string;
+    name?: string;
+    role?: string;
+    runtime?: string;
+}
+export interface AgentFlowInteraction {
+    id: string;
+    sequence: number;
+    sourceParticipantId: string;
+    targetParticipantId: string;
+    operation: string;
+    parentInteractionId?: string;
+    taskId?: string;
+    contextId?: string;
+    state?: string;
+    inputText?: string;
+    outputText?: string;
+    error?: string;
+    startedAt?: string;
+    endedAt?: string;
+}
+export interface AgentFlowTrace {
+    version: 1;
+    protocol?: string;
+    rootParticipantId: string;
+    participants: readonly AgentFlowParticipant[];
+    interactions: readonly AgentFlowInteraction[];
+    outcome: {
+        state: string;
+        outputText?: string;
+        error?: string;
+    };
+    completeness: AgentFlowEvidenceCompleteness;
+}
+export interface AgentFlowSummary {
+    participant_count: number;
+    interaction_count: number;
+    max_observed_depth: number;
+    root_state: string;
+    completeness: AgentFlowEvidenceCompleteness;
+}
+export declare function summarizeFlow(flow: AgentFlowTrace): AgentFlowSummary;

package/dist/sdk/agent-flow.js

@@ -0,0 +1,23 @@
+export function summarizeFlow(flow) {
+    const parents = new Map(flow.interactions.map(interaction => [interaction.id, interaction.parentInteractionId]));
+    const depth = (id) => {
+        let current = parents.get(id);
+        let value = 1;
+        const seen = new Set([id]);
+        while (current && !seen.has(current)) {
+            seen.add(current);
+            value += 1;
+            current = parents.get(current);
+        }
+        return value;
+    };
+    return {
+        participant_count: flow.participants.length,
+        interaction_count: flow.interactions.length,
+        max_observed_depth: flow.interactions.length > 0
+            ? Math.max(...flow.interactions.map(interaction => depth(interaction.id)))
+            : 0,
+        root_state: flow.outcome.state,
+        completeness: structuredClone(flow.completeness),
+    };
+}

package/dist/sdk/agent.js

@@ -20,6 +20,8 @@ import { collectOpenCodeMcpToolNames } from '../agents/opencode/contract.js';
 import { collectSensitiveEnvValues } from '../tool-event-results.js';
 import { createAskUserHandler } from './ask-bus/handler.js';
 import { resolveAgentRuntimeOptions } from './agent-runtime-options.js';
+import { extractToolEventsFromLog } from '../tool-events.js';
+import { clearToolEventRuntimeMetadata } from './tool-event-secrets.js';
 /**
  * Test-only injection point: override the sink used by the next emitter
  * built inside `createAgent`. Pass `null` to restore the default (stderr).
@@ -392,7 +394,14 @@ class AgentImpl {
             }
         }
         finally {
-            await this.ws.dispose();
+            try {
+                await this.ws.dispose();
+            }
+            finally {
+                for (const event of extractToolEventsFromLog(this._log)) {
+                    clearToolEventRuntimeMetadata(event);
+                }
+            }
         }
     }
 }

package/dist/sdk/evaluate.d.ts

@@ -1,4 +1,4 @@
-import type { Agent, Scorer, RecordedEvalResult, ScorerResultEntry, EvaluateOptions } from './types.js';
+import type { Agent, Scorer, RecordedEvalResult, ScorerResultEntry, EvaluateOptions, AgentFlowTrace } from './types.js';
 export type OnScorerErrorMode = NonNullable<EvaluateOptions['onScorerError']>;
 export declare class EvalScorerError extends Error {
     readonly scorerErrors: ScorerResultEntry[];
@@ -9,6 +9,7 @@ type EvaluateFromSnapshot = (snapshotPath: string, scorers: Scorer[], opts?: Eva
 type EvaluateFn = ((agent: Agent, scorers: Scorer[], opts?: EvaluateOptions) => Promise<RecordedEvalResult>) & {
     fromSnapshot: EvaluateFromSnapshot;
 };
+export declare function evaluateFlow(flow: AgentFlowTrace, scorers: Scorer[], opts?: EvaluateOptions): Promise<RecordedEvalResult>;
 /** Internal runConversation hook: step scorers must not consume final-run attribution. */
 export declare function evaluateStepScorers(agent: Agent, scorers: Scorer[], opts?: EvaluateOptions): Promise<RecordedEvalResult>;
 export declare const evaluate: EvaluateFn;

package/dist/sdk/evaluate.js

@@ -1,14 +1,18 @@
 import { countShellCommandsFromLog, extractSkillsFromLog, extractToolEventsFromLog } from '../tool-events.js';
 import { createScorerRevision } from '../reporting/comparison-contract.js';
 import { getRuntime } from './eval-runtime.js';
-import { emitEvalResult } from './result-capture.js';
+import { emitEvalResult, emitFlowEvalResult } from './result-capture.js';
 import { runJudgePipeline } from './judge-pipeline.js';
 import { runScorer } from './run-scorer.js';
 import { createLLMClient } from '../utils/llm.js';
 import { sandboxExec } from '../providers/sandbox-exec.js';
 import { buildTranscript, loadRunSnapshot, WorkspaceMissingError } from './snapshots.js';
+import { sanitizePersistenceValue } from '../tool-event-results.js';
+import { getOriginalMcpInput } from './mcp-event-input.js';
+import { cloneToolEventWithRuntimeMetadata, collectToolEventSensitiveValues, } from './tool-event-secrets.js';
 import fs from 'fs-extra';
 import path from 'path';
+import { summarizeFlow } from './agent-flow.js';
 export class EvalScorerError extends Error {
     scorerErrors;
     result;
@@ -19,6 +23,60 @@ export class EvalScorerError extends Error {
         this.scorerErrors = scorerErrors;
     }
 }
+export async function evaluateFlow(flow, scorers, opts) {
+    if (scorers.some(scorer => scorer.type === 'tool_usage')) {
+        throw new TypeError('tool_usage scorers require an Agent and cannot evaluate an AgentFlowTrace');
+    }
+    if (scorers.some(scorer => scorer.type === 'judge' && scorer.tools && scorer.tools.length > 0)) {
+        throw new TypeError('tool-enabled judge scorers require an Agent workspace and cannot evaluate an AgentFlowTrace');
+    }
+    const trackedLLM = opts?.llm ?? createLLMClient({ adapters: [{
+                name: 'runtime', isAvailable: async () => true,
+                call: (prompt, callOpts) => getRuntime().llm.call(prompt, callOpts),
+            }] });
+    const unavailable = async () => {
+        throw new WorkspaceMissingError('AgentFlowTrace does not include a workspace');
+    };
+    const ctx = {
+        workspace: '',
+        log: [],
+        transcript: JSON.stringify(flow),
+        toolEvents: [],
+        runCommand: unavailable,
+        artifacts: { list: () => [], read: unavailable, latest: async () => null },
+        flow,
+    };
+    const scoringStartedAt = performance.now();
+    const beforeTokenUsage = trackedLLM.tokenUsage ?? { inputTokens: 0, outputTokens: 0 };
+    const { result: evalResult, tokens: deltaTokenUsage } = trackedLLM.measure
+        ? await trackedLLM.measure(() => evaluateWithContext(ctx, scorers, { ...opts, llm: trackedLLM }))
+        : await (async () => {
+            const result = await evaluateWithContext(ctx, scorers, { ...opts, llm: trackedLLM });
+            const afterTokenUsage = trackedLLM.tokenUsage ?? beforeTokenUsage;
+            return { result, tokens: {
+                    inputTokens: Math.max(0, afterTokenUsage.inputTokens - beforeTokenUsage.inputTokens),
+                    outputTokens: Math.max(0, afterTokenUsage.outputTokens - beforeTokenUsage.outputTokens),
+                } };
+        })();
+    const scorerRevision = createScorerRevision(scorers);
+    const recordedResult = {
+        ...evalResult,
+        tokenUsage: deltaTokenUsage,
+        resultKind: 'evaluated',
+        scoringDurationMs: Math.max(0, performance.now() - scoringStartedAt),
+        recordedAt: new Date().toISOString(),
+        ...(opts?.evaluationDefinitionKey ? { evaluationDefinitionKey: opts.evaluationDefinitionKey } : {}),
+        ...(scorerRevision ? { scorerRevision } : {}),
+        trial: {
+            ...buildTrialResult([], { ...evalResult, tokenUsage: deltaTokenUsage }),
+            flow_summary: summarizeFlow(flow),
+            flow_trace: structuredClone(flow),
+        },
+    };
+    emitFlowEvalResult({ result: recordedResult, flow });
+    maybeThrowOnScorerErrors(recordedResult, opts?.onScorerError ?? 'skip');
+    return recordedResult;
+}
 /**
  * Run scorers against a trial and compute a weighted average score.
  *
@@ -99,6 +157,9 @@ export function evaluateStepScorers(agent, scorers, opts) {
     return evaluateAgent(agent, scorers, opts, false);
 }
 async function fromSnapshot(snapshotPath, scorers, opts) {
+    if (opts?.deterministicToolEvidence === 'live') {
+        throw new TypeError("evaluate.fromSnapshot() does not support deterministicToolEvidence: 'live'");
+    }
     const snapshot = await loadRunSnapshot(snapshotPath);
     const trackedLLM = opts?.llm ?? createLLMClient({ adapters: [{
                 name: 'runtime', isAvailable: async () => true,
@@ -146,6 +207,9 @@ async function evaluateWithContext(ctx, scorers, opts) {
                 call: (prompt, callOpts) => getRuntime().llm.call(prompt, callOpts),
             }] });
     const onScorerError = opts?.onScorerError ?? 'skip';
+    const deterministicCtx = opts?.deterministicToolEvidence === 'live'
+        ? createLiveDeterministicContext(ctx)
+        : ctx;
     const phase1 = [];
     const phase2 = [];
     const phase3 = [];
@@ -164,7 +228,7 @@ async function evaluateWithContext(ctx, scorers, opts) {
         }
     }
     const results = [];
-    const phase1Results = await Promise.all(phase1.map((g) => runScorer(g, ctx)));
+    const phase1Results = await Promise.all(phase1.map((g) => runScorer(g, deterministicCtx)));
     results.push(...phase1Results);
     const anyCheckFailed = failFast && phase1Results.some((r) => r.type === 'check' && r.status !== 'error' && r.score === 0);
     if (anyCheckFailed) {
@@ -175,7 +239,7 @@ async function evaluateWithContext(ctx, scorers, opts) {
     else {
         const judgeResults = await runJudgePipeline(phase2, ctx, { llm: trackedLLM });
         results.push(...judgeResults);
-        const phase3Results = await Promise.all(phase3.map((g) => runScorer(g, ctx)));
+        const phase3Results = await Promise.all(phase3.map((g) => runScorer(g, deterministicCtx)));
         results.push(...phase3Results);
     }
     const tokenUsage = trackedLLM.tokenUsage ?? { inputTokens: 0, outputTokens: 0 };
@@ -185,12 +249,58 @@ async function evaluateWithContext(ctx, scorers, opts) {
     const totalWeight = scoringResults.reduce((sum, r) => sum + r.weight, 0);
     const weightedSum = scoringResults.reduce((sum, r) => sum + r.score * r.weight, 0);
     const score = totalWeight > 0 ? weightedSum / totalWeight : 0;
-    return {
+    return sanitizePersistenceValue({
         score,
         scorers: results,
         tokenUsage,
+    }, collectToolEventSensitiveValues(ctx.toolEvents));
+}
+function createLiveDeterministicContext(ctx) {
+    return {
+        ...ctx,
+        toolEvents: ctx.toolEvents.map((event) => {
+            const originalInput = event.action === 'mcp_tool_call'
+                ? getOriginalMcpInput(event)
+                : undefined;
+            const mcp = normalizedMcpClassification(event);
+            return originalInput
+                ? cloneToolEventWithRuntimeMetadata(event, { arguments: {
+                        ...event.arguments,
+                        server: event.arguments?.server,
+                        tool: event.arguments?.tool,
+                        status: event.arguments?.status,
+                        ...structuredClone(originalInput),
+                    }, ...(mcp ? { mcp } : {}) })
+                : event;
+        }),
     };
 }
+function normalizedMcpClassification(event) {
+    if (event.mcp)
+        return event.mcp;
+    const args = event.arguments ?? {};
+    const separator = event.providerToolName.indexOf('.');
+    const serverName = typeof args.server === 'string' ? args.server
+        : separator > 0 ? event.providerToolName.slice(0, separator) : undefined;
+    const toolName = typeof args.tool === 'string' ? args.tool
+        : separator > 0 && separator < event.providerToolName.length - 1
+            ? event.providerToolName.slice(separator + 1) : undefined;
+    const status = event.status ?? args.status;
+    if (!serverName || !toolName)
+        return undefined;
+    if (status === 'completed')
+        return { serverName, toolName, invocation: 'confirmed', outcome: 'completed' };
+    if (status === 'error' || status === 'failed') {
+        return { serverName, toolName, invocation: 'confirmed', outcome: 'tool_error' };
+    }
+    if (status === 'user_denied' || status === 'policy_denied') {
+        return { serverName, toolName, invocation: 'not_invoked', outcome: status };
+    }
+    if (status === 'protocol_error') {
+        return { serverName, toolName, invocation: 'unknown', outcome: 'protocol_error' };
+    }
+    return undefined;
+}
 function maybeThrowOnScorerErrors(result, mode) {
     if (mode !== 'fail')
         return;

package/dist/sdk/index.d.ts

@@ -9,7 +9,7 @@ export type { McpMockApprovalRule, McpMockJsonValue } from './mcp-mock-approvals
 export type { CompileScenarioResult, EvidenceEnvelope, JsonObject, JsonPrimitive, JsonValue, MatchExpr, MatchPredicate, McpContent, ScenarioArtifact, ScenarioCase, ScenarioDiagnostic, ScenarioEvidence, ScenarioMachineV2, ScenarioRejection, ScenarioSuccess, ScenarioToolError, } from './scenario-machine-v2.js';
 export type { ExpectedScenarioCaseMatch, ScenarioCaseMatchEvidence, ScenarioStateTransitionEvidence, } from './scenario-evidence.js';
 export { decideMcpToolCall, redactMcpSecrets, } from './mcp-safety.js';
-export { evaluate, EvalScorerError } from './evaluate.js';
+export { evaluate, evaluateFlow, EvalScorerError } from './evaluate.js';
 export { RUN_SNAPSHOT_VERSION, buildRunSnapshot, loadRunSnapshot, SnapshotParseError, SnapshotVersionError, WorkspaceMissingError, } from './snapshots.js';
 export { createPersona } from './persona.js';
 export { createConversationWindow } from './conversation-window.js';
@@ -26,11 +26,13 @@ export { createAskBus, requireAskBusForLiveBatches, AskBusTimeoutError } from '.
 export { toAskUserToolEvent } from './ask-bus/projection.js';
 export type { AskUserToolEvent, AskUserToolEventArguments, AskUserToolEventQuestionArgument, } from './ask-bus/projection.js';
 export { buildAskBatchLogEntries } from './agent-result-log.js';
-export { emitEvalResult, resetAllResultObserversForTests, resetUserResultObservers, subscribeToEvalResults, } from './result-capture.js';
+export { emitEvalResult, emitFlowEvalResult, resetAllResultObserversForTests, resetUserResultObservers, subscribeToEvalResults, subscribeToEvaluationResults, } from './result-capture.js';
 export { getAgentCapabilities } from './types.js';
 export type { AgentTransport, AgentCapabilities, AgentName, McpRunMode, McpSafetyOptions, McpToolPolicy, McpToolPolicyRule, } from './types.js';
 export type { AskBus, AskBatch, AskQuestion, AskOption, AskAnswer, AskResolution, AskBatchSnapshot, AskAnswerSnapshot, AskResolutionSnapshot, AskHandle, AskHandler, AskSource, AskLifecycle, AskAnswerSource, Unsubscribe as AskBusUnsubscribe, } from './ask-bus/types.js';
 export type { Agent, AgentOptions, DebugOptions, Message, Scorer, CheckScorer, ScoreScorer, JudgeScorer, ToolUsageScorer, ScorerContext, EvalResult, ScorerResultEntry, ScorerStatus, ChatSession, ConversationResult, ConverseOptions, UntilPredicate, UntilContext, Reaction, TextReaction, AskUserReaction, AskUserQuestion, AskUserOption, ReactionPreviewEntry, TextReactionPreviewEntry, AskUserReactionPreviewEntry, ReactionPreviewResult, ReactionPreviewTurn, StepScorer, Persona, PersonaConfig, ConversationWindowConfig, TurnDetail, ReactionFiredEntry, PathgradePluginOptions, PathgradeMeta, TurnTiming, TokenUsage, EvaluateOptions, ReactionPreviewStatus, ScoreResult, JudgeInput, CodeJudgeToolName, ToolExpectation, SessionArtifactMatchOptions, SessionArtifactContent, SessionArtifacts, RecordedEvalResult, PathgradeTestMeta, EvaluationResultKind, AgentExecutionMetadata, AgentExecutionTransport, AgentInteractionMode, } from './types.js';
+export type { AgentFlowTrace, AgentFlowParticipant, AgentFlowInteraction, AgentFlowEvidenceAvailability, AgentFlowEvidenceCompleteness, AgentFlowSummary, } from './agent-flow.js';
+export type { EvaluationResultEvent, EvaluationResultObserver } from './result-capture.js';
 export type { ConversationWindow, ConversationWindowOptions } from './conversation-window.js';
 export type { JudgePipelineOptions } from './judge-pipeline.js';
 export type { RunScorerOptions } from './run-scorer.js';

package/dist/sdk/index.js

@@ -7,7 +7,7 @@ export { compileScenario, compileScenarioText } from './scenario-machine-v2.js';
 export { findScenarioCaseMatches, getFinalScenarioState, getScenarioStateTimeline, ScenarioEvidenceError, wasScenarioCaseMatched, } from './scenario-evidence.js';
 export { getMcpToolCall, isMcpToolCall, findMcpToolCalls, getMcpStartupStatus, isMcpStartupStatus, getMcpApproval, isMcpApproval, findMcpApprovals, getMcpInvocation, wasMcpToolInvoked, } from './mcp-evidence.js';
 export { decideMcpToolCall, redactMcpSecrets, } from './mcp-safety.js';
-export { evaluate, EvalScorerError } from './evaluate.js';
+export { evaluate, evaluateFlow, EvalScorerError } from './evaluate.js';
 export { RUN_SNAPSHOT_VERSION, buildRunSnapshot, loadRunSnapshot, SnapshotParseError, SnapshotVersionError, WorkspaceMissingError, } from './snapshots.js';
 export { createPersona } from './persona.js';
 export { createConversationWindow } from './conversation-window.js';
@@ -23,7 +23,7 @@ export { parsePathgradeReport } from '../reporting/report-parser.js';
 export { createAskBus, requireAskBusForLiveBatches, AskBusTimeoutError } from './ask-bus/bus.js';
 export { toAskUserToolEvent } from './ask-bus/projection.js';
 export { buildAskBatchLogEntries } from './agent-result-log.js';
-export { emitEvalResult, resetAllResultObserversForTests, resetUserResultObservers, subscribeToEvalResults, } from './result-capture.js';
+export { emitEvalResult, emitFlowEvalResult, resetAllResultObserversForTests, resetUserResultObservers, subscribeToEvalResults, subscribeToEvaluationResults, } from './result-capture.js';
 export { getAgentCapabilities } from './types.js';
 export { COMPARISON_CONTRACT_VERSION, NORMALIZED_RUN_MODEL_VERSION, PATHGRADE_REPORT_VERSION, TASK_INVENTORY_VERSION, } from '../reporting/reliability-contract.js';
 export { createAgentLLM, createLLMClient, ProviderNotSupportedError } from '../utils/llm.js';

package/dist/sdk/judge-prompt-builder.js

@@ -1,13 +1,15 @@
 import { sanitizePersistenceValue } from '../tool-event-results.js';
-import { getToolEventSensitiveValues } from './tool-event-secrets.js';
+import { collectToolEventSensitiveValues, } from './tool-event-secrets.js';
 export function buildJudgePrompt(scorer, ctx, input) {
     const sections = [];
-    sections.push(`## Session Transcript\n${ctx.transcript}`);
+    const sensitiveValues = collectToolEventSensitiveValues(ctx.toolEvents);
+    sections.push(`## Session Transcript\n${sanitizePersistenceValue(ctx.transcript, sensitiveValues)}`);
     if (scorer.includeToolEvents && ctx.toolEvents.length > 0) {
         sections.push(`## Tool Events\n${formatToolEvents(ctx)}`);
     }
     if (input) {
-        for (const [key, value] of Object.entries(input)) {
+        const sanitizedInput = sanitizePersistenceValue(input, sensitiveValues);
+        for (const [key, value] of Object.entries(sanitizedInput)) {
             const body = typeof value === 'string' ? value : JSON.stringify(value, null, 2);
             sections.push(`## ${key}\n${body}`);
         }
@@ -22,10 +24,10 @@ ${scorer.rubric}
 Respond with ONLY a JSON object: {"score": <number>, "details": "<brief explanation>"}`;
 }
 function formatToolEvents(ctx) {
+    const sensitiveValues = collectToolEventSensitiveValues(ctx.toolEvents);
     return ctx.toolEvents
         .map((event) => {
         const turn = event.turnNumber ? `turn ${event.turnNumber}` : 'instruction';
-        const sensitiveValues = getToolEventSensitiveValues(event);
         const details = [
             event.status ? `  status: ${event.status}` : undefined,
             event.mcp ? `  mcp: ${JSON.stringify(event.mcp)}` : undefined,
@@ -41,7 +43,8 @@ function formatToolEvents(ctx) {
 }
 export function buildBatchedJudgePrompt(judges, ctx, inputs) {
     const sections = [];
-    sections.push(`## Session Transcript\n${ctx.transcript}`);
+    const sensitiveValues = collectToolEventSensitiveValues(ctx.toolEvents);
+    sections.push(`## Session Transcript\n${sanitizePersistenceValue(ctx.transcript, sensitiveValues)}`);
     if (judges.some((j) => j.includeToolEvents) && ctx.toolEvents.length > 0) {
         sections.push(`## Tool Events\n${formatToolEvents(ctx)}`);
     }
@@ -49,7 +52,8 @@ export function buildBatchedJudgePrompt(judges, ctx, inputs) {
         const parts = [`### Rubric ${i + 1}: "${j.name}"\n${j.rubric}`];
         const input = inputs[i];
         if (input) {
-            for (const [key, value] of Object.entries(input)) {
+            const sanitizedInput = sanitizePersistenceValue(input, sensitiveValues);
+            for (const [key, value] of Object.entries(sanitizedInput)) {
                 const body = typeof value === 'string' ? value : JSON.stringify(value, null, 2);
                 parts.push(`#### ${key}\n${body}`);
             }
@@ -77,7 +81,7 @@ export function buildToolUseJudgePrompt(scorer, ctx) {
     ].join('\n');
     const parts = [];
     parts.push('## Session Transcript');
-    parts.push(ctx.transcript);
+    parts.push(sanitizePersistenceValue(ctx.transcript, collectToolEventSensitiveValues(ctx.toolEvents)));
     if (scorer.includeToolEvents && ctx.toolEvents.length > 0) {
         parts.push('## Tool Events');
         parts.push(formatToolEvents(ctx));